1 /* Aes.c -- AES encryption / decryption
2 2024-03-01 : Igor Pavlov : Public domain */
3
4 #include "Precomp.h"
5
6 #include "CpuArch.h"
7 #include "Aes.h"
8
9 AES_CODE_FUNC g_AesCbc_Decode;
10 #ifndef Z7_SFX
11 AES_CODE_FUNC g_AesCbc_Encode;
12 AES_CODE_FUNC g_AesCtr_Code;
13 UInt32 g_Aes_SupportedFunctions_Flags;
14 #endif
15
16 MY_ALIGN(64)
17 static UInt32 T[256 * 4];
18 MY_ALIGN(64)
19 static const Byte Sbox[256] = {
20 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76,
21 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0, 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0,
22 0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc, 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15,
23 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a, 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75,
24 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0, 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84,
25 0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b, 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf,
26 0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85, 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8,
27 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5, 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2,
28 0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17, 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73,
29 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88, 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb,
30 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c, 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79,
31 0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9, 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08,
32 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6, 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a,
33 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e, 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e,
34 0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94, 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf,
35 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16};
36
37
38 MY_ALIGN(64)
39 static UInt32 D[256 * 4];
40 MY_ALIGN(64)
41 static Byte InvS[256];
42
43 #define xtime(x) ((((x) << 1) ^ (((x) & 0x80) != 0 ? 0x1B : 0)) & 0xFF)
44
45 #define Ui32(a0, a1, a2, a3) ((UInt32)(a0) | ((UInt32)(a1) << 8) | ((UInt32)(a2) << 16) | ((UInt32)(a3) << 24))
46
47 #define gb0(x) ( (x) & 0xFF)
48 #define gb1(x) (((x) >> ( 8)) & 0xFF)
49 #define gb2(x) (((x) >> (16)) & 0xFF)
50 #define gb3(x) (((x) >> (24)))
51
52 #define gb(n, x) gb ## n(x)
53
54 #define TT(x) (T + (x << 8))
55 #define DD(x) (D + (x << 8))
56
57
58 // #define Z7_SHOW_AES_STATUS
59
60 #ifdef MY_CPU_X86_OR_AMD64
61
62 #if defined(__INTEL_COMPILER)
63 #if (__INTEL_COMPILER >= 1110)
64 #define USE_HW_AES
65 #if (__INTEL_COMPILER >= 1900)
66 #define USE_HW_VAES
67 #endif
68 #endif
69 #elif defined(Z7_CLANG_VERSION) && (Z7_CLANG_VERSION >= 30800) \
70 || defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 40400)
71 #define USE_HW_AES
72 #if defined(__clang__) && (__clang_major__ >= 8) \
73 || defined(__GNUC__) && (__GNUC__ >= 8)
74 #define USE_HW_VAES
75 #endif
76 #elif defined(_MSC_VER)
77 #define USE_HW_AES
78 #define USE_HW_VAES
79 #endif
80
81 #elif defined(MY_CPU_ARM_OR_ARM64) && defined(MY_CPU_LE)
82
83 #if defined(__ARM_FEATURE_AES) \
84 || defined(__ARM_FEATURE_CRYPTO)
85 #define USE_HW_AES
86 #else
87 #if defined(MY_CPU_ARM64) \
88 || defined(__ARM_ARCH) && (__ARM_ARCH >= 4) \
89 || defined(Z7_MSC_VER_ORIGINAL)
90 #if defined(__ARM_FP) && \
91 ( defined(Z7_CLANG_VERSION) && (Z7_CLANG_VERSION >= 30800) \
92 || defined(__GNUC__) && (__GNUC__ >= 6) \
93 ) \
94 || defined(Z7_MSC_VER_ORIGINAL) && (_MSC_VER >= 1910)
95 #if defined(MY_CPU_ARM64) \
96 || !defined(Z7_CLANG_VERSION) \
97 || defined(__ARM_NEON) && \
98 (Z7_CLANG_VERSION < 170000 || \
99 Z7_CLANG_VERSION > 170001)
100 #define USE_HW_AES
101 #endif
102 #endif
103 #endif
104 #endif
105 #endif
106
107 #ifdef USE_HW_AES
108 // #pragma message("=== Aes.c USE_HW_AES === ")
109 #ifdef Z7_SHOW_AES_STATUS
110 #include <stdio.h>
111 #define PRF(x) x
112 #else
113 #define PRF(x)
114 #endif
115 #endif
116
117
AesGenTables(void)118 void AesGenTables(void)
119 {
120 unsigned i;
121 for (i = 0; i < 256; i++)
122 InvS[Sbox[i]] = (Byte)i;
123
124 for (i = 0; i < 256; i++)
125 {
126 {
127 const UInt32 a1 = Sbox[i];
128 const UInt32 a2 = xtime(a1);
129 const UInt32 a3 = a2 ^ a1;
130 TT(0)[i] = Ui32(a2, a1, a1, a3);
131 TT(1)[i] = Ui32(a3, a2, a1, a1);
132 TT(2)[i] = Ui32(a1, a3, a2, a1);
133 TT(3)[i] = Ui32(a1, a1, a3, a2);
134 }
135 {
136 const UInt32 a1 = InvS[i];
137 const UInt32 a2 = xtime(a1);
138 const UInt32 a4 = xtime(a2);
139 const UInt32 a8 = xtime(a4);
140 const UInt32 a9 = a8 ^ a1;
141 const UInt32 aB = a8 ^ a2 ^ a1;
142 const UInt32 aD = a8 ^ a4 ^ a1;
143 const UInt32 aE = a8 ^ a4 ^ a2;
144 DD(0)[i] = Ui32(aE, a9, aD, aB);
145 DD(1)[i] = Ui32(aB, aE, a9, aD);
146 DD(2)[i] = Ui32(aD, aB, aE, a9);
147 DD(3)[i] = Ui32(a9, aD, aB, aE);
148 }
149 }
150
151 {
152 AES_CODE_FUNC d = AesCbc_Decode;
153 #ifndef Z7_SFX
154 AES_CODE_FUNC e = AesCbc_Encode;
155 AES_CODE_FUNC c = AesCtr_Code;
156 UInt32 flags = 0;
157 #endif
158
159 #ifdef USE_HW_AES
160 if (CPU_IsSupported_AES())
161 {
162 // #pragma message ("AES HW")
163 PRF(printf("\n===AES HW\n"));
164 d = AesCbc_Decode_HW;
165
166 #ifndef Z7_SFX
167 e = AesCbc_Encode_HW;
168 c = AesCtr_Code_HW;
169 flags = k_Aes_SupportedFunctions_HW;
170 #endif
171
172 #ifdef MY_CPU_X86_OR_AMD64
173 #ifdef USE_HW_VAES
174 if (CPU_IsSupported_VAES_AVX2())
175 {
176 PRF(printf("\n===vaes avx2\n"));
177 d = AesCbc_Decode_HW_256;
178 #ifndef Z7_SFX
179 c = AesCtr_Code_HW_256;
180 flags |= k_Aes_SupportedFunctions_HW_256;
181 #endif
182 }
183 #endif
184 #endif
185 }
186 #endif
187
188 g_AesCbc_Decode = d;
189 #ifndef Z7_SFX
190 g_AesCbc_Encode = e;
191 g_AesCtr_Code = c;
192 g_Aes_SupportedFunctions_Flags = flags;
193 #endif
194 }
195 }
196
197
198 #define HT(i, x, s) TT(x)[gb(x, s[(i + x) & 3])]
199
200 #define HT4(m, i, s, p) m[i] = \
201 HT(i, 0, s) ^ \
202 HT(i, 1, s) ^ \
203 HT(i, 2, s) ^ \
204 HT(i, 3, s) ^ w[p + i]
205
206 #define HT16(m, s, p) \
207 HT4(m, 0, s, p); \
208 HT4(m, 1, s, p); \
209 HT4(m, 2, s, p); \
210 HT4(m, 3, s, p); \
211
212 #define FT(i, x) Sbox[gb(x, m[(i + x) & 3])]
213 #define FT4(i) dest[i] = Ui32(FT(i, 0), FT(i, 1), FT(i, 2), FT(i, 3)) ^ w[i];
214
215
216 #define HD(i, x, s) DD(x)[gb(x, s[(i - x) & 3])]
217
218 #define HD4(m, i, s, p) m[i] = \
219 HD(i, 0, s) ^ \
220 HD(i, 1, s) ^ \
221 HD(i, 2, s) ^ \
222 HD(i, 3, s) ^ w[p + i];
223
224 #define HD16(m, s, p) \
225 HD4(m, 0, s, p); \
226 HD4(m, 1, s, p); \
227 HD4(m, 2, s, p); \
228 HD4(m, 3, s, p); \
229
230 #define FD(i, x) InvS[gb(x, m[(i - x) & 3])]
231 #define FD4(i) dest[i] = Ui32(FD(i, 0), FD(i, 1), FD(i, 2), FD(i, 3)) ^ w[i];
232
Aes_SetKey_Enc(UInt32 * w,const Byte * key,unsigned keySize)233 void Z7_FASTCALL Aes_SetKey_Enc(UInt32 *w, const Byte *key, unsigned keySize)
234 {
235 unsigned i, m;
236 const UInt32 *wLim;
237 UInt32 t;
238 UInt32 rcon = 1;
239
240 keySize /= 4;
241 w[0] = ((UInt32)keySize / 2) + 3;
242 w += 4;
243
244 for (i = 0; i < keySize; i++, key += 4)
245 w[i] = GetUi32(key);
246
247 t = w[(size_t)keySize - 1];
248 wLim = w + (size_t)keySize * 3 + 28;
249 m = 0;
250 do
251 {
252 if (m == 0)
253 {
254 t = Ui32(Sbox[gb1(t)] ^ rcon, Sbox[gb2(t)], Sbox[gb3(t)], Sbox[gb0(t)]);
255 rcon <<= 1;
256 if (rcon & 0x100)
257 rcon = 0x1b;
258 m = keySize;
259 }
260 else if (m == 4 && keySize > 6)
261 t = Ui32(Sbox[gb0(t)], Sbox[gb1(t)], Sbox[gb2(t)], Sbox[gb3(t)]);
262 m--;
263 t ^= w[0];
264 w[keySize] = t;
265 }
266 while (++w != wLim);
267 }
268
Aes_SetKey_Dec(UInt32 * w,const Byte * key,unsigned keySize)269 void Z7_FASTCALL Aes_SetKey_Dec(UInt32 *w, const Byte *key, unsigned keySize)
270 {
271 unsigned i, num;
272 Aes_SetKey_Enc(w, key, keySize);
273 num = keySize + 20;
274 w += 8;
275 for (i = 0; i < num; i++)
276 {
277 UInt32 r = w[i];
278 w[i] =
279 DD(0)[Sbox[gb0(r)]] ^
280 DD(1)[Sbox[gb1(r)]] ^
281 DD(2)[Sbox[gb2(r)]] ^
282 DD(3)[Sbox[gb3(r)]];
283 }
284 }
285
286 /* Aes_Encode and Aes_Decode functions work with little-endian words.
287 src and dest are pointers to 4 UInt32 words.
288 src and dest can point to same block */
289
290 // Z7_FORCE_INLINE
Aes_Encode(const UInt32 * w,UInt32 * dest,const UInt32 * src)291 static void Aes_Encode(const UInt32 *w, UInt32 *dest, const UInt32 *src)
292 {
293 UInt32 s[4];
294 UInt32 m[4];
295 UInt32 numRounds2 = w[0];
296 w += 4;
297 s[0] = src[0] ^ w[0];
298 s[1] = src[1] ^ w[1];
299 s[2] = src[2] ^ w[2];
300 s[3] = src[3] ^ w[3];
301 w += 4;
302 for (;;)
303 {
304 HT16(m, s, 0)
305 if (--numRounds2 == 0)
306 break;
307 HT16(s, m, 4)
308 w += 8;
309 }
310 w += 4;
311 FT4(0)
312 FT4(1)
313 FT4(2)
314 FT4(3)
315 }
316
317 Z7_FORCE_INLINE
Aes_Decode(const UInt32 * w,UInt32 * dest,const UInt32 * src)318 static void Aes_Decode(const UInt32 *w, UInt32 *dest, const UInt32 *src)
319 {
320 UInt32 s[4];
321 UInt32 m[4];
322 UInt32 numRounds2 = w[0];
323 w += 4 + numRounds2 * 8;
324 s[0] = src[0] ^ w[0];
325 s[1] = src[1] ^ w[1];
326 s[2] = src[2] ^ w[2];
327 s[3] = src[3] ^ w[3];
328 for (;;)
329 {
330 w -= 8;
331 HD16(m, s, 4)
332 if (--numRounds2 == 0)
333 break;
334 HD16(s, m, 0)
335 }
336 FD4(0)
337 FD4(1)
338 FD4(2)
339 FD4(3)
340 }
341
AesCbc_Init(UInt32 * p,const Byte * iv)342 void AesCbc_Init(UInt32 *p, const Byte *iv)
343 {
344 unsigned i;
345 for (i = 0; i < 4; i++)
346 p[i] = GetUi32(iv + i * 4);
347 }
348
AesCbc_Encode(UInt32 * p,Byte * data,size_t numBlocks)349 void Z7_FASTCALL AesCbc_Encode(UInt32 *p, Byte *data, size_t numBlocks)
350 {
351 for (; numBlocks != 0; numBlocks--, data += AES_BLOCK_SIZE)
352 {
353 p[0] ^= GetUi32(data);
354 p[1] ^= GetUi32(data + 4);
355 p[2] ^= GetUi32(data + 8);
356 p[3] ^= GetUi32(data + 12);
357
358 Aes_Encode(p + 4, p, p);
359
360 SetUi32(data, p[0])
361 SetUi32(data + 4, p[1])
362 SetUi32(data + 8, p[2])
363 SetUi32(data + 12, p[3])
364 }
365 }
366
AesCbc_Decode(UInt32 * p,Byte * data,size_t numBlocks)367 void Z7_FASTCALL AesCbc_Decode(UInt32 *p, Byte *data, size_t numBlocks)
368 {
369 UInt32 in[4], out[4];
370 for (; numBlocks != 0; numBlocks--, data += AES_BLOCK_SIZE)
371 {
372 in[0] = GetUi32(data);
373 in[1] = GetUi32(data + 4);
374 in[2] = GetUi32(data + 8);
375 in[3] = GetUi32(data + 12);
376
377 Aes_Decode(p + 4, out, in);
378
379 SetUi32(data, p[0] ^ out[0])
380 SetUi32(data + 4, p[1] ^ out[1])
381 SetUi32(data + 8, p[2] ^ out[2])
382 SetUi32(data + 12, p[3] ^ out[3])
383
384 p[0] = in[0];
385 p[1] = in[1];
386 p[2] = in[2];
387 p[3] = in[3];
388 }
389 }
390
AesCtr_Code(UInt32 * p,Byte * data,size_t numBlocks)391 void Z7_FASTCALL AesCtr_Code(UInt32 *p, Byte *data, size_t numBlocks)
392 {
393 for (; numBlocks != 0; numBlocks--)
394 {
395 UInt32 temp[4];
396 unsigned i;
397
398 if (++p[0] == 0)
399 p[1]++;
400
401 Aes_Encode(p + 4, temp, p);
402
403 for (i = 0; i < 4; i++, data += 4)
404 {
405 const UInt32 t = temp[i];
406
407 #ifdef MY_CPU_LE_UNALIGN
408 *((UInt32 *)(void *)data) ^= t;
409 #else
410 data[0] = (Byte)(data[0] ^ (t & 0xFF));
411 data[1] = (Byte)(data[1] ^ ((t >> 8) & 0xFF));
412 data[2] = (Byte)(data[2] ^ ((t >> 16) & 0xFF));
413 data[3] = (Byte)(data[3] ^ ((t >> 24)));
414 #endif
415 }
416 }
417 }
418
419 #undef xtime
420 #undef Ui32
421 #undef gb0
422 #undef gb1
423 #undef gb2
424 #undef gb3
425 #undef gb
426 #undef TT
427 #undef DD
428 #undef USE_HW_AES
429 #undef PRF
430