AESNI.c 9.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265
  1. /*
  2. * AESNI.c: AES using AES-NI instructions
  3. *
  4. * Written in 2013 by Sebastian Ramacher <sebastian@ramacher.at>
  5. *
  6. * ===================================================================
  7. * The contents of this file are dedicated to the public domain. To
  8. * the extent that dedication to the public domain is not available,
  9. * everyone is granted a worldwide, perpetual, royalty-free,
  10. * non-exclusive license to exercise all rights associated with the
  11. * contents of this file for any purpose whatsoever.
  12. * No rights are reserved.
  13. *
  14. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  15. * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  16. * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  17. * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  18. * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  19. * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  20. * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  21. * SOFTWARE.
  22. * ===================================================================
  23. */
  24. #include "pycrypto_common.h"
  25. #include "block_base.h"
  26. #include <wmmintrin.h>
  27. #include <stdlib.h>
  28. FAKE_INIT(raw_aesni)
  29. #define MODULE_NAME AESNI
  30. #define BLOCK_SIZE 16
  31. #define KEY_SIZE 0
  32. #define MAXKC (256/32)
  33. #define MAXKB (256/8)
  34. #define MAXNR 14
  35. #define ALIGNMENT 16
  36. typedef struct {
  37. /** Both ek and dk points into the buffer and are aligned to the 16 byte boundary **/
  38. __m128i* ek;
  39. __m128i* dk;
  40. int rounds;
  41. uint8_t buffer[(MAXNR+1)*sizeof(__m128i)*2 + ALIGNMENT];
  42. } block_state;
  43. /* Helper functions to expand keys */
  44. static __m128i aes128_keyexpand(__m128i key)
  45. {
  46. key = _mm_xor_si128(key, _mm_slli_si128(key, 4));
  47. key = _mm_xor_si128(key, _mm_slli_si128(key, 4));
  48. return _mm_xor_si128(key, _mm_slli_si128(key, 4));
  49. }
  50. static __m128i aes192_keyexpand_2(__m128i key, __m128i key2)
  51. {
  52. key = _mm_shuffle_epi32(key, 0xff);
  53. key2 = _mm_xor_si128(key2, _mm_slli_si128(key2, 4));
  54. return _mm_xor_si128(key, key2);
  55. }
  56. #define KEYEXP128_H(K1, K2, I, S) _mm_xor_si128(aes128_keyexpand(K1), \
  57. _mm_shuffle_epi32(_mm_aeskeygenassist_si128(K2, I), S))
  58. #define KEYEXP128(K, I) KEYEXP128_H(K, K, I, 0xff)
  59. #define KEYEXP192(K1, K2, I) KEYEXP128_H(K1, K2, I, 0x55)
  60. #define KEYEXP192_2(K1, K2) aes192_keyexpand_2(K1, K2)
  61. #define KEYEXP256(K1, K2, I) KEYEXP128_H(K1, K2, I, 0xff)
  62. #define KEYEXP256_2(K1, K2) KEYEXP128_H(K1, K2, 0x00, 0xaa)
  63. #define SHUFFLE128_0(a, b) \
  64. _mm_castpd_si128( \
  65. _mm_shuffle_pd( \
  66. _mm_castsi128_pd(a), \
  67. _mm_castsi128_pd(b), \
  68. 0))
  69. #define SHUFFLE128_1(a, b) \
  70. _mm_castpd_si128( \
  71. _mm_shuffle_pd( \
  72. _mm_castsi128_pd(a), \
  73. _mm_castsi128_pd(b), \
  74. 1))
  75. /* Encryption key setup */
  76. static void aes_key_setup_enc(__m128i *rk, const uint8_t* cipherKey, int keylen)
  77. {
  78. switch (keylen) {
  79. case 16:
  80. {
  81. /* 128 bit key setup */
  82. rk[0] = _mm_loadu_si128((const __m128i*) cipherKey);
  83. rk[1] = KEYEXP128(rk[0], 0x01);
  84. rk[2] = KEYEXP128(rk[1], 0x02);
  85. rk[3] = KEYEXP128(rk[2], 0x04);
  86. rk[4] = KEYEXP128(rk[3], 0x08);
  87. rk[5] = KEYEXP128(rk[4], 0x10);
  88. rk[6] = KEYEXP128(rk[5], 0x20);
  89. rk[7] = KEYEXP128(rk[6], 0x40);
  90. rk[8] = KEYEXP128(rk[7], 0x80);
  91. rk[9] = KEYEXP128(rk[8], 0x1B);
  92. rk[10] = KEYEXP128(rk[9], 0x36);
  93. break;
  94. }
  95. case 24:
  96. {
  97. /* 192 bit key setup */
  98. uint8_t key[24];
  99. __m128i temp[2];
  100. memcpy(key, cipherKey, 24);
  101. rk[0] = _mm_loadu_si128((const __m128i*) key);
  102. rk[1] = _mm_loadu_si128((const __m128i*) (key+16));
  103. temp[0] = KEYEXP192(rk[0], rk[1], 0x01);
  104. temp[1] = KEYEXP192_2(temp[0], rk[1]);
  105. rk[1] = SHUFFLE128_0(rk[1], temp[0]);
  106. rk[2] = SHUFFLE128_1(temp[0], temp[1]);
  107. rk[3] = KEYEXP192(temp[0], temp[1], 0x02);
  108. rk[4] = KEYEXP192_2(rk[3], temp[1]);
  109. temp[0] = KEYEXP192(rk[3], rk[4], 0x04);
  110. temp[1] = KEYEXP192_2(temp[0], rk[4]);
  111. rk[4] = SHUFFLE128_0(rk[4], temp[0]);
  112. rk[5] = SHUFFLE128_1(temp[0], temp[1]);
  113. rk[6] = KEYEXP192(temp[0], temp[1], 0x08);
  114. rk[7] = KEYEXP192_2(rk[6], temp[1]);
  115. temp[0] = KEYEXP192(rk[6], rk[7], 0x10);
  116. temp[1] = KEYEXP192_2(temp[0], rk[7]);
  117. rk[7] = SHUFFLE128_0(rk[7], temp[0]);
  118. rk[8] = SHUFFLE128_1(temp[0], temp[1]);
  119. rk[9] = KEYEXP192(temp[0], temp[1], 0x20);
  120. rk[10] = KEYEXP192_2(rk[9], temp[1]);
  121. temp[0] = KEYEXP192(rk[9], rk[10], 0x40);
  122. temp[1] = KEYEXP192_2(temp[0], rk[10]);
  123. rk[10] = SHUFFLE128_0(rk[10], temp[0]);
  124. rk[11] = SHUFFLE128_1(temp[0], temp[1]);
  125. rk[12] = KEYEXP192(temp[0], temp[1], 0x80);
  126. break;
  127. }
  128. case 32:
  129. {
  130. /* 256 bit key setup */
  131. rk[0] = _mm_loadu_si128((const __m128i*) cipherKey);
  132. rk[1] = _mm_loadu_si128((const __m128i*) (cipherKey+16));
  133. rk[2] = KEYEXP256(rk[0], rk[1], 0x01);
  134. rk[3] = KEYEXP256_2(rk[1], rk[2]);
  135. rk[4] = KEYEXP256(rk[2], rk[3], 0x02);
  136. rk[5] = KEYEXP256_2(rk[3], rk[4]);
  137. rk[6] = KEYEXP256(rk[4], rk[5], 0x04);
  138. rk[7] = KEYEXP256_2(rk[5], rk[6]);
  139. rk[8] = KEYEXP256(rk[6], rk[7], 0x08);
  140. rk[9] = KEYEXP256_2(rk[7], rk[8]);
  141. rk[10] = KEYEXP256(rk[8], rk[9], 0x10);
  142. rk[11] = KEYEXP256_2(rk[9], rk[10]);
  143. rk[12] = KEYEXP256(rk[10], rk[11], 0x20);
  144. rk[13] = KEYEXP256_2(rk[11], rk[12]);
  145. rk[14] = KEYEXP256(rk[12], rk[13], 0x40);
  146. break;
  147. }
  148. }
  149. }
  150. /* Decryption key setup */
  151. static void aes_key_setup_dec(__m128i *dk, const __m128i *ek, int rounds)
  152. {
  153. int i;
  154. dk[rounds] = ek[0];
  155. for (i = 1; i < rounds; ++i) {
  156. dk[rounds - i] = _mm_aesimc_si128(ek[i]);
  157. }
  158. dk[0] = ek[rounds];
  159. }
  160. static int block_init(block_state* self, unsigned char* key, int keylen)
  161. {
  162. int nr = 0;
  163. int offset;
  164. switch (keylen) {
  165. case 16: nr = 10; break;
  166. case 24: nr = 12; break;
  167. case 32: nr = 14; break;
  168. default:
  169. return ERR_NR_ROUNDS;
  170. }
  171. /* ensure that self->ek and self->dk are aligned to 16 byte boundaries */
  172. offset = ALIGNMENT - ((uintptr_t)self->buffer & (ALIGNMENT-1));
  173. self->ek = (__m128i*)((uint8_t*)self->buffer + offset);
  174. self->dk = (__m128i*)((uint8_t*)self->ek + (MAXNR+1)*sizeof(__m128i));
  175. self->rounds = nr;
  176. aes_key_setup_enc(self->ek, key, keylen);
  177. aes_key_setup_dec(self->dk, self->ek, nr);
  178. return 0;
  179. }
  180. static void block_finalize(block_state* self)
  181. {
  182. memset(self, 0, sizeof(*self));
  183. }
  184. static void block_encrypt(block_state* self, const uint8_t* in, uint8_t* out)
  185. {
  186. __m128i m = _mm_loadu_si128((const __m128i*) in);
  187. /* first 9 rounds */
  188. m = _mm_xor_si128(m, self->ek[0]);
  189. m = _mm_aesenc_si128(m, self->ek[1]);
  190. m = _mm_aesenc_si128(m, self->ek[2]);
  191. m = _mm_aesenc_si128(m, self->ek[3]);
  192. m = _mm_aesenc_si128(m, self->ek[4]);
  193. m = _mm_aesenc_si128(m, self->ek[5]);
  194. m = _mm_aesenc_si128(m, self->ek[6]);
  195. m = _mm_aesenc_si128(m, self->ek[7]);
  196. m = _mm_aesenc_si128(m, self->ek[8]);
  197. m = _mm_aesenc_si128(m, self->ek[9]);
  198. if (self->rounds != 10) {
  199. /* two additional rounds for AES-192/256 */
  200. m = _mm_aesenc_si128(m, self->ek[10]);
  201. m = _mm_aesenc_si128(m, self->ek[11]);
  202. if (self->rounds == 14) {
  203. /* another two additional rounds for AES-256 */
  204. m = _mm_aesenc_si128(m, self->ek[12]);
  205. m = _mm_aesenc_si128(m, self->ek[13]);
  206. }
  207. }
  208. m = _mm_aesenclast_si128(m, self->ek[self->rounds]);
  209. _mm_storeu_si128((__m128i*) out, m);
  210. }
  211. static void block_decrypt(block_state* self, const uint8_t* in, uint8_t* out)
  212. {
  213. __m128i m = _mm_loadu_si128((const __m128i*) in);
  214. /* first 9 rounds */
  215. m = _mm_xor_si128(m, self->dk[0]);
  216. m = _mm_aesdec_si128(m, self->dk[1]);
  217. m = _mm_aesdec_si128(m, self->dk[2]);
  218. m = _mm_aesdec_si128(m, self->dk[3]);
  219. m = _mm_aesdec_si128(m, self->dk[4]);
  220. m = _mm_aesdec_si128(m, self->dk[5]);
  221. m = _mm_aesdec_si128(m, self->dk[6]);
  222. m = _mm_aesdec_si128(m, self->dk[7]);
  223. m = _mm_aesdec_si128(m, self->dk[8]);
  224. m = _mm_aesdec_si128(m, self->dk[9]);
  225. if (self->rounds != 10) {
  226. /* two additional rounds for AES-192/256 */
  227. m = _mm_aesdec_si128(m, self->dk[10]);
  228. m = _mm_aesdec_si128(m, self->dk[11]);
  229. if (self->rounds == 14) {
  230. /* another two additional rounds for AES-256 */
  231. m = _mm_aesdec_si128(m, self->dk[12]);
  232. m = _mm_aesdec_si128(m, self->dk[13]);
  233. }
  234. }
  235. m = _mm_aesdeclast_si128(m, self->dk[self->rounds]);
  236. _mm_storeu_si128((__m128i*) out, m);
  237. }
  238. #include "block_common.c"