crypto: aesni-intel - Ported implementation to x86-32
[linux-2.6.git] / arch / x86 / crypto / aesni-intel_glue.c
1 /*
2  * Support for Intel AES-NI instructions. This file contains glue
3  * code, the real AES implementation is in intel-aes_asm.S.
4  *
5  * Copyright (C) 2008, Intel Corp.
6  *    Author: Huang Ying <ying.huang@intel.com>
7  *
8  * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
9  * interface for 64-bit kernels.
10  *    Authors: Adrian Hoban <adrian.hoban@intel.com>
11  *             Gabriele Paoloni <gabriele.paoloni@intel.com>
12  *             Tadeusz Struk (tadeusz.struk@intel.com)
13  *             Aidan O'Mahony (aidan.o.mahony@intel.com)
14  *    Copyright (c) 2010, Intel Corporation.
15  *
16  * This program is free software; you can redistribute it and/or modify
17  * it under the terms of the GNU General Public License as published by
18  * the Free Software Foundation; either version 2 of the License, or
19  * (at your option) any later version.
20  */
21
22 #include <linux/hardirq.h>
23 #include <linux/types.h>
24 #include <linux/crypto.h>
25 #include <linux/err.h>
26 #include <crypto/algapi.h>
27 #include <crypto/aes.h>
28 #include <crypto/cryptd.h>
29 #include <crypto/ctr.h>
30 #include <asm/i387.h>
31 #include <asm/aes.h>
32 #include <crypto/scatterwalk.h>
33 #include <crypto/internal/aead.h>
34 #include <linux/workqueue.h>
35 #include <linux/spinlock.h>
36
37 #if defined(CONFIG_CRYPTO_CTR) || defined(CONFIG_CRYPTO_CTR_MODULE)
38 #define HAS_CTR
39 #endif
40
41 #if defined(CONFIG_CRYPTO_LRW) || defined(CONFIG_CRYPTO_LRW_MODULE)
42 #define HAS_LRW
43 #endif
44
45 #if defined(CONFIG_CRYPTO_PCBC) || defined(CONFIG_CRYPTO_PCBC_MODULE)
46 #define HAS_PCBC
47 #endif
48
49 #if defined(CONFIG_CRYPTO_XTS) || defined(CONFIG_CRYPTO_XTS_MODULE)
50 #define HAS_XTS
51 #endif
52
53 struct async_aes_ctx {
54         struct cryptd_ablkcipher *cryptd_tfm;
55 };
56
57 /* This data is stored at the end of the crypto_tfm struct.
58  * It's a type of per "session" data storage location.
59  * This needs to be 16 byte aligned.
60  */
61 struct aesni_rfc4106_gcm_ctx {
62         u8 hash_subkey[16];
63         struct crypto_aes_ctx aes_key_expanded;
64         u8 nonce[4];
65         struct cryptd_aead *cryptd_tfm;
66 };
67
68 struct aesni_gcm_set_hash_subkey_result {
69         int err;
70         struct completion completion;
71 };
72
73 struct aesni_hash_subkey_req_data {
74         u8 iv[16];
75         struct aesni_gcm_set_hash_subkey_result result;
76         struct scatterlist sg;
77 };
78
79 #define AESNI_ALIGN     (16)
80 #define AES_BLOCK_MASK  (~(AES_BLOCK_SIZE-1))
81 #define RFC4106_HASH_SUBKEY_SIZE 16
82
83 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
84                              unsigned int key_len);
85 asmlinkage void aesni_enc(struct crypto_aes_ctx *ctx, u8 *out,
86                           const u8 *in);
87 asmlinkage void aesni_dec(struct crypto_aes_ctx *ctx, u8 *out,
88                           const u8 *in);
89 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out,
90                               const u8 *in, unsigned int len);
91 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out,
92                               const u8 *in, unsigned int len);
93 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
94                               const u8 *in, unsigned int len, u8 *iv);
95 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
96                               const u8 *in, unsigned int len, u8 *iv);
97 #ifdef CONFIG_X86_64
98 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out,
99                               const u8 *in, unsigned int len, u8 *iv);
100 #endif
101
102 /* asmlinkage void aesni_gcm_enc()
103  * void *ctx,  AES Key schedule. Starts on a 16 byte boundary.
104  * u8 *out, Ciphertext output. Encrypt in-place is allowed.
105  * const u8 *in, Plaintext input
106  * unsigned long plaintext_len, Length of data in bytes for encryption.
107  * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association)
108  *         concatenated with 8 byte Initialisation Vector (from IPSec ESP
109  *         Payload) concatenated with 0x00000001. 16-byte aligned pointer.
110  * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
111  * const u8 *aad, Additional Authentication Data (AAD)
112  * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this
113  *          is going to be 8 or 12 bytes
114  * u8 *auth_tag, Authenticated Tag output.
115  * unsigned long auth_tag_len), Authenticated Tag Length in bytes.
116  *          Valid values are 16 (most likely), 12 or 8.
117  */
118 asmlinkage void aesni_gcm_enc(void *ctx, u8 *out,
119                         const u8 *in, unsigned long plaintext_len, u8 *iv,
120                         u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
121                         u8 *auth_tag, unsigned long auth_tag_len);
122
123 /* asmlinkage void aesni_gcm_dec()
124  * void *ctx, AES Key schedule. Starts on a 16 byte boundary.
125  * u8 *out, Plaintext output. Decrypt in-place is allowed.
126  * const u8 *in, Ciphertext input
127  * unsigned long ciphertext_len, Length of data in bytes for decryption.
128  * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association)
129  *         concatenated with 8 byte Initialisation Vector (from IPSec ESP
130  *         Payload) concatenated with 0x00000001. 16-byte aligned pointer.
131  * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
132  * const u8 *aad, Additional Authentication Data (AAD)
133  * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this is going
134  * to be 8 or 12 bytes
135  * u8 *auth_tag, Authenticated Tag output.
136  * unsigned long auth_tag_len) Authenticated Tag Length in bytes.
137  * Valid values are 16 (most likely), 12 or 8.
138  */
139 asmlinkage void aesni_gcm_dec(void *ctx, u8 *out,
140                         const u8 *in, unsigned long ciphertext_len, u8 *iv,
141                         u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
142                         u8 *auth_tag, unsigned long auth_tag_len);
143
144 static inline struct
145 aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm)
146 {
147         return
148                 (struct aesni_rfc4106_gcm_ctx *)
149                 PTR_ALIGN((u8 *)
150                 crypto_tfm_ctx(crypto_aead_tfm(tfm)), AESNI_ALIGN);
151 }
152
153 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
154 {
155         unsigned long addr = (unsigned long)raw_ctx;
156         unsigned long align = AESNI_ALIGN;
157
158         if (align <= crypto_tfm_ctx_alignment())
159                 align = 1;
160         return (struct crypto_aes_ctx *)ALIGN(addr, align);
161 }
162
163 static int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx,
164                               const u8 *in_key, unsigned int key_len)
165 {
166         struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx);
167         u32 *flags = &tfm->crt_flags;
168         int err;
169
170         if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 &&
171             key_len != AES_KEYSIZE_256) {
172                 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
173                 return -EINVAL;
174         }
175
176         if (!irq_fpu_usable())
177                 err = crypto_aes_expand_key(ctx, in_key, key_len);
178         else {
179                 kernel_fpu_begin();
180                 err = aesni_set_key(ctx, in_key, key_len);
181                 kernel_fpu_end();
182         }
183
184         return err;
185 }
186
187 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
188                        unsigned int key_len)
189 {
190         return aes_set_key_common(tfm, crypto_tfm_ctx(tfm), in_key, key_len);
191 }
192
193 static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
194 {
195         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
196
197         if (!irq_fpu_usable())
198                 crypto_aes_encrypt_x86(ctx, dst, src);
199         else {
200                 kernel_fpu_begin();
201                 aesni_enc(ctx, dst, src);
202                 kernel_fpu_end();
203         }
204 }
205
206 static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
207 {
208         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
209
210         if (!irq_fpu_usable())
211                 crypto_aes_decrypt_x86(ctx, dst, src);
212         else {
213                 kernel_fpu_begin();
214                 aesni_dec(ctx, dst, src);
215                 kernel_fpu_end();
216         }
217 }
218
219 static struct crypto_alg aesni_alg = {
220         .cra_name               = "aes",
221         .cra_driver_name        = "aes-aesni",
222         .cra_priority           = 300,
223         .cra_flags              = CRYPTO_ALG_TYPE_CIPHER,
224         .cra_blocksize          = AES_BLOCK_SIZE,
225         .cra_ctxsize            = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
226         .cra_alignmask          = 0,
227         .cra_module             = THIS_MODULE,
228         .cra_list               = LIST_HEAD_INIT(aesni_alg.cra_list),
229         .cra_u  = {
230                 .cipher = {
231                         .cia_min_keysize        = AES_MIN_KEY_SIZE,
232                         .cia_max_keysize        = AES_MAX_KEY_SIZE,
233                         .cia_setkey             = aes_set_key,
234                         .cia_encrypt            = aes_encrypt,
235                         .cia_decrypt            = aes_decrypt
236                 }
237         }
238 };
239
240 static void __aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
241 {
242         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
243
244         aesni_enc(ctx, dst, src);
245 }
246
247 static void __aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
248 {
249         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
250
251         aesni_dec(ctx, dst, src);
252 }
253
254 static struct crypto_alg __aesni_alg = {
255         .cra_name               = "__aes-aesni",
256         .cra_driver_name        = "__driver-aes-aesni",
257         .cra_priority           = 0,
258         .cra_flags              = CRYPTO_ALG_TYPE_CIPHER,
259         .cra_blocksize          = AES_BLOCK_SIZE,
260         .cra_ctxsize            = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
261         .cra_alignmask          = 0,
262         .cra_module             = THIS_MODULE,
263         .cra_list               = LIST_HEAD_INIT(__aesni_alg.cra_list),
264         .cra_u  = {
265                 .cipher = {
266                         .cia_min_keysize        = AES_MIN_KEY_SIZE,
267                         .cia_max_keysize        = AES_MAX_KEY_SIZE,
268                         .cia_setkey             = aes_set_key,
269                         .cia_encrypt            = __aes_encrypt,
270                         .cia_decrypt            = __aes_decrypt
271                 }
272         }
273 };
274
275 static int ecb_encrypt(struct blkcipher_desc *desc,
276                        struct scatterlist *dst, struct scatterlist *src,
277                        unsigned int nbytes)
278 {
279         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
280         struct blkcipher_walk walk;
281         int err;
282
283         blkcipher_walk_init(&walk, dst, src, nbytes);
284         err = blkcipher_walk_virt(desc, &walk);
285         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
286
287         kernel_fpu_begin();
288         while ((nbytes = walk.nbytes)) {
289                 aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
290                               nbytes & AES_BLOCK_MASK);
291                 nbytes &= AES_BLOCK_SIZE - 1;
292                 err = blkcipher_walk_done(desc, &walk, nbytes);
293         }
294         kernel_fpu_end();
295
296         return err;
297 }
298
299 static int ecb_decrypt(struct blkcipher_desc *desc,
300                        struct scatterlist *dst, struct scatterlist *src,
301                        unsigned int nbytes)
302 {
303         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
304         struct blkcipher_walk walk;
305         int err;
306
307         blkcipher_walk_init(&walk, dst, src, nbytes);
308         err = blkcipher_walk_virt(desc, &walk);
309         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
310
311         kernel_fpu_begin();
312         while ((nbytes = walk.nbytes)) {
313                 aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
314                               nbytes & AES_BLOCK_MASK);
315                 nbytes &= AES_BLOCK_SIZE - 1;
316                 err = blkcipher_walk_done(desc, &walk, nbytes);
317         }
318         kernel_fpu_end();
319
320         return err;
321 }
322
323 static struct crypto_alg blk_ecb_alg = {
324         .cra_name               = "__ecb-aes-aesni",
325         .cra_driver_name        = "__driver-ecb-aes-aesni",
326         .cra_priority           = 0,
327         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
328         .cra_blocksize          = AES_BLOCK_SIZE,
329         .cra_ctxsize            = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
330         .cra_alignmask          = 0,
331         .cra_type               = &crypto_blkcipher_type,
332         .cra_module             = THIS_MODULE,
333         .cra_list               = LIST_HEAD_INIT(blk_ecb_alg.cra_list),
334         .cra_u = {
335                 .blkcipher = {
336                         .min_keysize    = AES_MIN_KEY_SIZE,
337                         .max_keysize    = AES_MAX_KEY_SIZE,
338                         .setkey         = aes_set_key,
339                         .encrypt        = ecb_encrypt,
340                         .decrypt        = ecb_decrypt,
341                 },
342         },
343 };
344
345 static int cbc_encrypt(struct blkcipher_desc *desc,
346                        struct scatterlist *dst, struct scatterlist *src,
347                        unsigned int nbytes)
348 {
349         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
350         struct blkcipher_walk walk;
351         int err;
352
353         blkcipher_walk_init(&walk, dst, src, nbytes);
354         err = blkcipher_walk_virt(desc, &walk);
355         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
356
357         kernel_fpu_begin();
358         while ((nbytes = walk.nbytes)) {
359                 aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
360                               nbytes & AES_BLOCK_MASK, walk.iv);
361                 nbytes &= AES_BLOCK_SIZE - 1;
362                 err = blkcipher_walk_done(desc, &walk, nbytes);
363         }
364         kernel_fpu_end();
365
366         return err;
367 }
368
369 static int cbc_decrypt(struct blkcipher_desc *desc,
370                        struct scatterlist *dst, struct scatterlist *src,
371                        unsigned int nbytes)
372 {
373         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
374         struct blkcipher_walk walk;
375         int err;
376
377         blkcipher_walk_init(&walk, dst, src, nbytes);
378         err = blkcipher_walk_virt(desc, &walk);
379         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
380
381         kernel_fpu_begin();
382         while ((nbytes = walk.nbytes)) {
383                 aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
384                               nbytes & AES_BLOCK_MASK, walk.iv);
385                 nbytes &= AES_BLOCK_SIZE - 1;
386                 err = blkcipher_walk_done(desc, &walk, nbytes);
387         }
388         kernel_fpu_end();
389
390         return err;
391 }
392
393 static struct crypto_alg blk_cbc_alg = {
394         .cra_name               = "__cbc-aes-aesni",
395         .cra_driver_name        = "__driver-cbc-aes-aesni",
396         .cra_priority           = 0,
397         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
398         .cra_blocksize          = AES_BLOCK_SIZE,
399         .cra_ctxsize            = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
400         .cra_alignmask          = 0,
401         .cra_type               = &crypto_blkcipher_type,
402         .cra_module             = THIS_MODULE,
403         .cra_list               = LIST_HEAD_INIT(blk_cbc_alg.cra_list),
404         .cra_u = {
405                 .blkcipher = {
406                         .min_keysize    = AES_MIN_KEY_SIZE,
407                         .max_keysize    = AES_MAX_KEY_SIZE,
408                         .setkey         = aes_set_key,
409                         .encrypt        = cbc_encrypt,
410                         .decrypt        = cbc_decrypt,
411                 },
412         },
413 };
414
415 #ifdef CONFIG_X86_64
416 static void ctr_crypt_final(struct crypto_aes_ctx *ctx,
417                             struct blkcipher_walk *walk)
418 {
419         u8 *ctrblk = walk->iv;
420         u8 keystream[AES_BLOCK_SIZE];
421         u8 *src = walk->src.virt.addr;
422         u8 *dst = walk->dst.virt.addr;
423         unsigned int nbytes = walk->nbytes;
424
425         aesni_enc(ctx, keystream, ctrblk);
426         crypto_xor(keystream, src, nbytes);
427         memcpy(dst, keystream, nbytes);
428         crypto_inc(ctrblk, AES_BLOCK_SIZE);
429 }
430
431 static int ctr_crypt(struct blkcipher_desc *desc,
432                      struct scatterlist *dst, struct scatterlist *src,
433                      unsigned int nbytes)
434 {
435         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
436         struct blkcipher_walk walk;
437         int err;
438
439         blkcipher_walk_init(&walk, dst, src, nbytes);
440         err = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE);
441         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
442
443         kernel_fpu_begin();
444         while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
445                 aesni_ctr_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
446                               nbytes & AES_BLOCK_MASK, walk.iv);
447                 nbytes &= AES_BLOCK_SIZE - 1;
448                 err = blkcipher_walk_done(desc, &walk, nbytes);
449         }
450         if (walk.nbytes) {
451                 ctr_crypt_final(ctx, &walk);
452                 err = blkcipher_walk_done(desc, &walk, 0);
453         }
454         kernel_fpu_end();
455
456         return err;
457 }
458
459 static struct crypto_alg blk_ctr_alg = {
460         .cra_name               = "__ctr-aes-aesni",
461         .cra_driver_name        = "__driver-ctr-aes-aesni",
462         .cra_priority           = 0,
463         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
464         .cra_blocksize          = 1,
465         .cra_ctxsize            = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
466         .cra_alignmask          = 0,
467         .cra_type               = &crypto_blkcipher_type,
468         .cra_module             = THIS_MODULE,
469         .cra_list               = LIST_HEAD_INIT(blk_ctr_alg.cra_list),
470         .cra_u = {
471                 .blkcipher = {
472                         .min_keysize    = AES_MIN_KEY_SIZE,
473                         .max_keysize    = AES_MAX_KEY_SIZE,
474                         .ivsize         = AES_BLOCK_SIZE,
475                         .setkey         = aes_set_key,
476                         .encrypt        = ctr_crypt,
477                         .decrypt        = ctr_crypt,
478                 },
479         },
480 };
481 #endif
482
483 static int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key,
484                         unsigned int key_len)
485 {
486         struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
487         struct crypto_ablkcipher *child = &ctx->cryptd_tfm->base;
488         int err;
489
490         crypto_ablkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
491         crypto_ablkcipher_set_flags(child, crypto_ablkcipher_get_flags(tfm)
492                                     & CRYPTO_TFM_REQ_MASK);
493         err = crypto_ablkcipher_setkey(child, key, key_len);
494         crypto_ablkcipher_set_flags(tfm, crypto_ablkcipher_get_flags(child)
495                                     & CRYPTO_TFM_RES_MASK);
496         return err;
497 }
498
499 static int ablk_encrypt(struct ablkcipher_request *req)
500 {
501         struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
502         struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
503
504         if (!irq_fpu_usable()) {
505                 struct ablkcipher_request *cryptd_req =
506                         ablkcipher_request_ctx(req);
507                 memcpy(cryptd_req, req, sizeof(*req));
508                 ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
509                 return crypto_ablkcipher_encrypt(cryptd_req);
510         } else {
511                 struct blkcipher_desc desc;
512                 desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm);
513                 desc.info = req->info;
514                 desc.flags = 0;
515                 return crypto_blkcipher_crt(desc.tfm)->encrypt(
516                         &desc, req->dst, req->src, req->nbytes);
517         }
518 }
519
520 static int ablk_decrypt(struct ablkcipher_request *req)
521 {
522         struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
523         struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
524
525         if (!irq_fpu_usable()) {
526                 struct ablkcipher_request *cryptd_req =
527                         ablkcipher_request_ctx(req);
528                 memcpy(cryptd_req, req, sizeof(*req));
529                 ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
530                 return crypto_ablkcipher_decrypt(cryptd_req);
531         } else {
532                 struct blkcipher_desc desc;
533                 desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm);
534                 desc.info = req->info;
535                 desc.flags = 0;
536                 return crypto_blkcipher_crt(desc.tfm)->decrypt(
537                         &desc, req->dst, req->src, req->nbytes);
538         }
539 }
540
541 static void ablk_exit(struct crypto_tfm *tfm)
542 {
543         struct async_aes_ctx *ctx = crypto_tfm_ctx(tfm);
544
545         cryptd_free_ablkcipher(ctx->cryptd_tfm);
546 }
547
548 static void ablk_init_common(struct crypto_tfm *tfm,
549                              struct cryptd_ablkcipher *cryptd_tfm)
550 {
551         struct async_aes_ctx *ctx = crypto_tfm_ctx(tfm);
552
553         ctx->cryptd_tfm = cryptd_tfm;
554         tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request) +
555                 crypto_ablkcipher_reqsize(&cryptd_tfm->base);
556 }
557
558 static int ablk_ecb_init(struct crypto_tfm *tfm)
559 {
560         struct cryptd_ablkcipher *cryptd_tfm;
561
562         cryptd_tfm = cryptd_alloc_ablkcipher("__driver-ecb-aes-aesni", 0, 0);
563         if (IS_ERR(cryptd_tfm))
564                 return PTR_ERR(cryptd_tfm);
565         ablk_init_common(tfm, cryptd_tfm);
566         return 0;
567 }
568
569 static struct crypto_alg ablk_ecb_alg = {
570         .cra_name               = "ecb(aes)",
571         .cra_driver_name        = "ecb-aes-aesni",
572         .cra_priority           = 400,
573         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
574         .cra_blocksize          = AES_BLOCK_SIZE,
575         .cra_ctxsize            = sizeof(struct async_aes_ctx),
576         .cra_alignmask          = 0,
577         .cra_type               = &crypto_ablkcipher_type,
578         .cra_module             = THIS_MODULE,
579         .cra_list               = LIST_HEAD_INIT(ablk_ecb_alg.cra_list),
580         .cra_init               = ablk_ecb_init,
581         .cra_exit               = ablk_exit,
582         .cra_u = {
583                 .ablkcipher = {
584                         .min_keysize    = AES_MIN_KEY_SIZE,
585                         .max_keysize    = AES_MAX_KEY_SIZE,
586                         .setkey         = ablk_set_key,
587                         .encrypt        = ablk_encrypt,
588                         .decrypt        = ablk_decrypt,
589                 },
590         },
591 };
592
593 static int ablk_cbc_init(struct crypto_tfm *tfm)
594 {
595         struct cryptd_ablkcipher *cryptd_tfm;
596
597         cryptd_tfm = cryptd_alloc_ablkcipher("__driver-cbc-aes-aesni", 0, 0);
598         if (IS_ERR(cryptd_tfm))
599                 return PTR_ERR(cryptd_tfm);
600         ablk_init_common(tfm, cryptd_tfm);
601         return 0;
602 }
603
604 static struct crypto_alg ablk_cbc_alg = {
605         .cra_name               = "cbc(aes)",
606         .cra_driver_name        = "cbc-aes-aesni",
607         .cra_priority           = 400,
608         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
609         .cra_blocksize          = AES_BLOCK_SIZE,
610         .cra_ctxsize            = sizeof(struct async_aes_ctx),
611         .cra_alignmask          = 0,
612         .cra_type               = &crypto_ablkcipher_type,
613         .cra_module             = THIS_MODULE,
614         .cra_list               = LIST_HEAD_INIT(ablk_cbc_alg.cra_list),
615         .cra_init               = ablk_cbc_init,
616         .cra_exit               = ablk_exit,
617         .cra_u = {
618                 .ablkcipher = {
619                         .min_keysize    = AES_MIN_KEY_SIZE,
620                         .max_keysize    = AES_MAX_KEY_SIZE,
621                         .ivsize         = AES_BLOCK_SIZE,
622                         .setkey         = ablk_set_key,
623                         .encrypt        = ablk_encrypt,
624                         .decrypt        = ablk_decrypt,
625                 },
626         },
627 };
628
629 #ifdef CONFIG_X86_64
630 static int ablk_ctr_init(struct crypto_tfm *tfm)
631 {
632         struct cryptd_ablkcipher *cryptd_tfm;
633
634         cryptd_tfm = cryptd_alloc_ablkcipher("__driver-ctr-aes-aesni", 0, 0);
635         if (IS_ERR(cryptd_tfm))
636                 return PTR_ERR(cryptd_tfm);
637         ablk_init_common(tfm, cryptd_tfm);
638         return 0;
639 }
640
641 static struct crypto_alg ablk_ctr_alg = {
642         .cra_name               = "ctr(aes)",
643         .cra_driver_name        = "ctr-aes-aesni",
644         .cra_priority           = 400,
645         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
646         .cra_blocksize          = 1,
647         .cra_ctxsize            = sizeof(struct async_aes_ctx),
648         .cra_alignmask          = 0,
649         .cra_type               = &crypto_ablkcipher_type,
650         .cra_module             = THIS_MODULE,
651         .cra_list               = LIST_HEAD_INIT(ablk_ctr_alg.cra_list),
652         .cra_init               = ablk_ctr_init,
653         .cra_exit               = ablk_exit,
654         .cra_u = {
655                 .ablkcipher = {
656                         .min_keysize    = AES_MIN_KEY_SIZE,
657                         .max_keysize    = AES_MAX_KEY_SIZE,
658                         .ivsize         = AES_BLOCK_SIZE,
659                         .setkey         = ablk_set_key,
660                         .encrypt        = ablk_encrypt,
661                         .decrypt        = ablk_encrypt,
662                         .geniv          = "chainiv",
663                 },
664         },
665 };
666
667 #ifdef HAS_CTR
668 static int ablk_rfc3686_ctr_init(struct crypto_tfm *tfm)
669 {
670         struct cryptd_ablkcipher *cryptd_tfm;
671
672         cryptd_tfm = cryptd_alloc_ablkcipher(
673                 "rfc3686(__driver-ctr-aes-aesni)", 0, 0);
674         if (IS_ERR(cryptd_tfm))
675                 return PTR_ERR(cryptd_tfm);
676         ablk_init_common(tfm, cryptd_tfm);
677         return 0;
678 }
679
680 static struct crypto_alg ablk_rfc3686_ctr_alg = {
681         .cra_name               = "rfc3686(ctr(aes))",
682         .cra_driver_name        = "rfc3686-ctr-aes-aesni",
683         .cra_priority           = 400,
684         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
685         .cra_blocksize          = 1,
686         .cra_ctxsize            = sizeof(struct async_aes_ctx),
687         .cra_alignmask          = 0,
688         .cra_type               = &crypto_ablkcipher_type,
689         .cra_module             = THIS_MODULE,
690         .cra_list               = LIST_HEAD_INIT(ablk_rfc3686_ctr_alg.cra_list),
691         .cra_init               = ablk_rfc3686_ctr_init,
692         .cra_exit               = ablk_exit,
693         .cra_u = {
694                 .ablkcipher = {
695                         .min_keysize = AES_MIN_KEY_SIZE+CTR_RFC3686_NONCE_SIZE,
696                         .max_keysize = AES_MAX_KEY_SIZE+CTR_RFC3686_NONCE_SIZE,
697                         .ivsize      = CTR_RFC3686_IV_SIZE,
698                         .setkey      = ablk_set_key,
699                         .encrypt     = ablk_encrypt,
700                         .decrypt     = ablk_decrypt,
701                         .geniv       = "seqiv",
702                 },
703         },
704 };
705 #endif
706 #endif
707
708 #ifdef HAS_LRW
709 static int ablk_lrw_init(struct crypto_tfm *tfm)
710 {
711         struct cryptd_ablkcipher *cryptd_tfm;
712
713         cryptd_tfm = cryptd_alloc_ablkcipher("fpu(lrw(__driver-aes-aesni))",
714                                              0, 0);
715         if (IS_ERR(cryptd_tfm))
716                 return PTR_ERR(cryptd_tfm);
717         ablk_init_common(tfm, cryptd_tfm);
718         return 0;
719 }
720
721 static struct crypto_alg ablk_lrw_alg = {
722         .cra_name               = "lrw(aes)",
723         .cra_driver_name        = "lrw-aes-aesni",
724         .cra_priority           = 400,
725         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
726         .cra_blocksize          = AES_BLOCK_SIZE,
727         .cra_ctxsize            = sizeof(struct async_aes_ctx),
728         .cra_alignmask          = 0,
729         .cra_type               = &crypto_ablkcipher_type,
730         .cra_module             = THIS_MODULE,
731         .cra_list               = LIST_HEAD_INIT(ablk_lrw_alg.cra_list),
732         .cra_init               = ablk_lrw_init,
733         .cra_exit               = ablk_exit,
734         .cra_u = {
735                 .ablkcipher = {
736                         .min_keysize    = AES_MIN_KEY_SIZE + AES_BLOCK_SIZE,
737                         .max_keysize    = AES_MAX_KEY_SIZE + AES_BLOCK_SIZE,
738                         .ivsize         = AES_BLOCK_SIZE,
739                         .setkey         = ablk_set_key,
740                         .encrypt        = ablk_encrypt,
741                         .decrypt        = ablk_decrypt,
742                 },
743         },
744 };
745 #endif
746
747 #ifdef HAS_PCBC
748 static int ablk_pcbc_init(struct crypto_tfm *tfm)
749 {
750         struct cryptd_ablkcipher *cryptd_tfm;
751
752         cryptd_tfm = cryptd_alloc_ablkcipher("fpu(pcbc(__driver-aes-aesni))",
753                                              0, 0);
754         if (IS_ERR(cryptd_tfm))
755                 return PTR_ERR(cryptd_tfm);
756         ablk_init_common(tfm, cryptd_tfm);
757         return 0;
758 }
759
760 static struct crypto_alg ablk_pcbc_alg = {
761         .cra_name               = "pcbc(aes)",
762         .cra_driver_name        = "pcbc-aes-aesni",
763         .cra_priority           = 400,
764         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
765         .cra_blocksize          = AES_BLOCK_SIZE,
766         .cra_ctxsize            = sizeof(struct async_aes_ctx),
767         .cra_alignmask          = 0,
768         .cra_type               = &crypto_ablkcipher_type,
769         .cra_module             = THIS_MODULE,
770         .cra_list               = LIST_HEAD_INIT(ablk_pcbc_alg.cra_list),
771         .cra_init               = ablk_pcbc_init,
772         .cra_exit               = ablk_exit,
773         .cra_u = {
774                 .ablkcipher = {
775                         .min_keysize    = AES_MIN_KEY_SIZE,
776                         .max_keysize    = AES_MAX_KEY_SIZE,
777                         .ivsize         = AES_BLOCK_SIZE,
778                         .setkey         = ablk_set_key,
779                         .encrypt        = ablk_encrypt,
780                         .decrypt        = ablk_decrypt,
781                 },
782         },
783 };
784 #endif
785
786 #ifdef HAS_XTS
787 static int ablk_xts_init(struct crypto_tfm *tfm)
788 {
789         struct cryptd_ablkcipher *cryptd_tfm;
790
791         cryptd_tfm = cryptd_alloc_ablkcipher("fpu(xts(__driver-aes-aesni))",
792                                              0, 0);
793         if (IS_ERR(cryptd_tfm))
794                 return PTR_ERR(cryptd_tfm);
795         ablk_init_common(tfm, cryptd_tfm);
796         return 0;
797 }
798
799 static struct crypto_alg ablk_xts_alg = {
800         .cra_name               = "xts(aes)",
801         .cra_driver_name        = "xts-aes-aesni",
802         .cra_priority           = 400,
803         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
804         .cra_blocksize          = AES_BLOCK_SIZE,
805         .cra_ctxsize            = sizeof(struct async_aes_ctx),
806         .cra_alignmask          = 0,
807         .cra_type               = &crypto_ablkcipher_type,
808         .cra_module             = THIS_MODULE,
809         .cra_list               = LIST_HEAD_INIT(ablk_xts_alg.cra_list),
810         .cra_init               = ablk_xts_init,
811         .cra_exit               = ablk_exit,
812         .cra_u = {
813                 .ablkcipher = {
814                         .min_keysize    = 2 * AES_MIN_KEY_SIZE,
815                         .max_keysize    = 2 * AES_MAX_KEY_SIZE,
816                         .ivsize         = AES_BLOCK_SIZE,
817                         .setkey         = ablk_set_key,
818                         .encrypt        = ablk_encrypt,
819                         .decrypt        = ablk_decrypt,
820                 },
821         },
822 };
823 #endif
824
825 static int rfc4106_init(struct crypto_tfm *tfm)
826 {
827         struct cryptd_aead *cryptd_tfm;
828         struct aesni_rfc4106_gcm_ctx *ctx = (struct aesni_rfc4106_gcm_ctx *)
829                 PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN);
830         cryptd_tfm = cryptd_alloc_aead("__driver-gcm-aes-aesni", 0, 0);
831         if (IS_ERR(cryptd_tfm))
832                 return PTR_ERR(cryptd_tfm);
833         ctx->cryptd_tfm = cryptd_tfm;
834         tfm->crt_aead.reqsize = sizeof(struct aead_request)
835                 + crypto_aead_reqsize(&cryptd_tfm->base);
836         return 0;
837 }
838
839 static void rfc4106_exit(struct crypto_tfm *tfm)
840 {
841         struct aesni_rfc4106_gcm_ctx *ctx =
842                 (struct aesni_rfc4106_gcm_ctx *)
843                 PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN);
844         if (!IS_ERR(ctx->cryptd_tfm))
845                 cryptd_free_aead(ctx->cryptd_tfm);
846         return;
847 }
848
849 static void
850 rfc4106_set_hash_subkey_done(struct crypto_async_request *req, int err)
851 {
852         struct aesni_gcm_set_hash_subkey_result *result = req->data;
853
854         if (err == -EINPROGRESS)
855                 return;
856         result->err = err;
857         complete(&result->completion);
858 }
859
860 static int
861 rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len)
862 {
863         struct crypto_ablkcipher *ctr_tfm;
864         struct ablkcipher_request *req;
865         int ret = -EINVAL;
866         struct aesni_hash_subkey_req_data *req_data;
867
868         ctr_tfm = crypto_alloc_ablkcipher("ctr(aes)", 0, 0);
869         if (IS_ERR(ctr_tfm))
870                 return PTR_ERR(ctr_tfm);
871
872         crypto_ablkcipher_clear_flags(ctr_tfm, ~0);
873
874         ret = crypto_ablkcipher_setkey(ctr_tfm, key, key_len);
875         if (ret) {
876                 crypto_free_ablkcipher(ctr_tfm);
877                 return ret;
878         }
879
880         req = ablkcipher_request_alloc(ctr_tfm, GFP_KERNEL);
881         if (!req) {
882                 crypto_free_ablkcipher(ctr_tfm);
883                 return -EINVAL;
884         }
885
886         req_data = kmalloc(sizeof(*req_data), GFP_KERNEL);
887         if (!req_data) {
888                 crypto_free_ablkcipher(ctr_tfm);
889                 return -ENOMEM;
890         }
891         memset(req_data->iv, 0, sizeof(req_data->iv));
892
893         /* Clear the data in the hash sub key container to zero.*/
894         /* We want to cipher all zeros to create the hash sub key. */
895         memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE);
896
897         init_completion(&req_data->result.completion);
898         sg_init_one(&req_data->sg, hash_subkey, RFC4106_HASH_SUBKEY_SIZE);
899         ablkcipher_request_set_tfm(req, ctr_tfm);
900         ablkcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP |
901                                         CRYPTO_TFM_REQ_MAY_BACKLOG,
902                                         rfc4106_set_hash_subkey_done,
903                                         &req_data->result);
904
905         ablkcipher_request_set_crypt(req, &req_data->sg,
906                 &req_data->sg, RFC4106_HASH_SUBKEY_SIZE, req_data->iv);
907
908         ret = crypto_ablkcipher_encrypt(req);
909         if (ret == -EINPROGRESS || ret == -EBUSY) {
910                 ret = wait_for_completion_interruptible
911                         (&req_data->result.completion);
912                 if (!ret)
913                         ret = req_data->result.err;
914         }
915         ablkcipher_request_free(req);
916         kfree(req_data);
917         crypto_free_ablkcipher(ctr_tfm);
918         return ret;
919 }
920
921 static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key,
922                                                    unsigned int key_len)
923 {
924         int ret = 0;
925         struct crypto_tfm *tfm = crypto_aead_tfm(parent);
926         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent);
927         u8 *new_key_mem = NULL;
928
929         if (key_len < 4) {
930                 crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
931                 return -EINVAL;
932         }
933         /*Account for 4 byte nonce at the end.*/
934         key_len -= 4;
935         if (key_len != AES_KEYSIZE_128) {
936                 crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
937                 return -EINVAL;
938         }
939
940         memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce));
941         /*This must be on a 16 byte boundary!*/
942         if ((unsigned long)(&(ctx->aes_key_expanded.key_enc[0])) % AESNI_ALIGN)
943                 return -EINVAL;
944
945         if ((unsigned long)key % AESNI_ALIGN) {
946                 /*key is not aligned: use an auxuliar aligned pointer*/
947                 new_key_mem = kmalloc(key_len+AESNI_ALIGN, GFP_KERNEL);
948                 if (!new_key_mem)
949                         return -ENOMEM;
950
951                 new_key_mem = PTR_ALIGN(new_key_mem, AESNI_ALIGN);
952                 memcpy(new_key_mem, key, key_len);
953                 key = new_key_mem;
954         }
955
956         if (!irq_fpu_usable())
957                 ret = crypto_aes_expand_key(&(ctx->aes_key_expanded),
958                 key, key_len);
959         else {
960                 kernel_fpu_begin();
961                 ret = aesni_set_key(&(ctx->aes_key_expanded), key, key_len);
962                 kernel_fpu_end();
963         }
964         /*This must be on a 16 byte boundary!*/
965         if ((unsigned long)(&(ctx->hash_subkey[0])) % AESNI_ALIGN) {
966                 ret = -EINVAL;
967                 goto exit;
968         }
969         ret = rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
970 exit:
971         kfree(new_key_mem);
972         return ret;
973 }
974
975 /* This is the Integrity Check Value (aka the authentication tag length and can
976  * be 8, 12 or 16 bytes long. */
977 static int rfc4106_set_authsize(struct crypto_aead *parent,
978                                 unsigned int authsize)
979 {
980         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent);
981         struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
982
983         switch (authsize) {
984         case 8:
985         case 12:
986         case 16:
987                 break;
988         default:
989                 return -EINVAL;
990         }
991         crypto_aead_crt(parent)->authsize = authsize;
992         crypto_aead_crt(cryptd_child)->authsize = authsize;
993         return 0;
994 }
995
996 static int rfc4106_encrypt(struct aead_request *req)
997 {
998         int ret;
999         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1000         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
1001         struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
1002
1003         if (!irq_fpu_usable()) {
1004                 struct aead_request *cryptd_req =
1005                         (struct aead_request *) aead_request_ctx(req);
1006                 memcpy(cryptd_req, req, sizeof(*req));
1007                 aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
1008                 return crypto_aead_encrypt(cryptd_req);
1009         } else {
1010                 kernel_fpu_begin();
1011                 ret = cryptd_child->base.crt_aead.encrypt(req);
1012                 kernel_fpu_end();
1013                 return ret;
1014         }
1015 }
1016
1017 static int rfc4106_decrypt(struct aead_request *req)
1018 {
1019         int ret;
1020         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1021         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
1022         struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
1023
1024         if (!irq_fpu_usable()) {
1025                 struct aead_request *cryptd_req =
1026                         (struct aead_request *) aead_request_ctx(req);
1027                 memcpy(cryptd_req, req, sizeof(*req));
1028                 aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
1029                 return crypto_aead_decrypt(cryptd_req);
1030         } else {
1031                 kernel_fpu_begin();
1032                 ret = cryptd_child->base.crt_aead.decrypt(req);
1033                 kernel_fpu_end();
1034                 return ret;
1035         }
1036 }
1037
1038 static struct crypto_alg rfc4106_alg = {
1039         .cra_name = "rfc4106(gcm(aes))",
1040         .cra_driver_name = "rfc4106-gcm-aesni",
1041         .cra_priority = 400,
1042         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
1043         .cra_blocksize = 1,
1044         .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx) + AESNI_ALIGN,
1045         .cra_alignmask = 0,
1046         .cra_type = &crypto_nivaead_type,
1047         .cra_module = THIS_MODULE,
1048         .cra_list = LIST_HEAD_INIT(rfc4106_alg.cra_list),
1049         .cra_init = rfc4106_init,
1050         .cra_exit = rfc4106_exit,
1051         .cra_u = {
1052                 .aead = {
1053                         .setkey = rfc4106_set_key,
1054                         .setauthsize = rfc4106_set_authsize,
1055                         .encrypt = rfc4106_encrypt,
1056                         .decrypt = rfc4106_decrypt,
1057                         .geniv = "seqiv",
1058                         .ivsize = 8,
1059                         .maxauthsize = 16,
1060                 },
1061         },
1062 };
1063
1064 static int __driver_rfc4106_encrypt(struct aead_request *req)
1065 {
1066         u8 one_entry_in_sg = 0;
1067         u8 *src, *dst, *assoc;
1068         __be32 counter = cpu_to_be32(1);
1069         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1070         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
1071         void *aes_ctx = &(ctx->aes_key_expanded);
1072         unsigned long auth_tag_len = crypto_aead_authsize(tfm);
1073         u8 iv_tab[16+AESNI_ALIGN];
1074         u8* iv = (u8 *) PTR_ALIGN((u8 *)iv_tab, AESNI_ALIGN);
1075         struct scatter_walk src_sg_walk;
1076         struct scatter_walk assoc_sg_walk;
1077         struct scatter_walk dst_sg_walk;
1078         unsigned int i;
1079
1080         /* Assuming we are supporting rfc4106 64-bit extended */
1081         /* sequence numbers We need to have the AAD length equal */
1082         /* to 8 or 12 bytes */
1083         if (unlikely(req->assoclen != 8 && req->assoclen != 12))
1084                 return -EINVAL;
1085         /* IV below built */
1086         for (i = 0; i < 4; i++)
1087                 *(iv+i) = ctx->nonce[i];
1088         for (i = 0; i < 8; i++)
1089                 *(iv+4+i) = req->iv[i];
1090         *((__be32 *)(iv+12)) = counter;
1091
1092         if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) {
1093                 one_entry_in_sg = 1;
1094                 scatterwalk_start(&src_sg_walk, req->src);
1095                 scatterwalk_start(&assoc_sg_walk, req->assoc);
1096                 src = scatterwalk_map(&src_sg_walk, 0);
1097                 assoc = scatterwalk_map(&assoc_sg_walk, 0);
1098                 dst = src;
1099                 if (unlikely(req->src != req->dst)) {
1100                         scatterwalk_start(&dst_sg_walk, req->dst);
1101                         dst = scatterwalk_map(&dst_sg_walk, 0);
1102                 }
1103
1104         } else {
1105                 /* Allocate memory for src, dst, assoc */
1106                 src = kmalloc(req->cryptlen + auth_tag_len + req->assoclen,
1107                         GFP_ATOMIC);
1108                 if (unlikely(!src))
1109                         return -ENOMEM;
1110                 assoc = (src + req->cryptlen + auth_tag_len);
1111                 scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0);
1112                 scatterwalk_map_and_copy(assoc, req->assoc, 0,
1113                                         req->assoclen, 0);
1114                 dst = src;
1115         }
1116
1117         aesni_gcm_enc(aes_ctx, dst, src, (unsigned long)req->cryptlen, iv,
1118                 ctx->hash_subkey, assoc, (unsigned long)req->assoclen, dst
1119                 + ((unsigned long)req->cryptlen), auth_tag_len);
1120
1121         /* The authTag (aka the Integrity Check Value) needs to be written
1122          * back to the packet. */
1123         if (one_entry_in_sg) {
1124                 if (unlikely(req->src != req->dst)) {
1125                         scatterwalk_unmap(dst, 0);
1126                         scatterwalk_done(&dst_sg_walk, 0, 0);
1127                 }
1128                 scatterwalk_unmap(src, 0);
1129                 scatterwalk_unmap(assoc, 0);
1130                 scatterwalk_done(&src_sg_walk, 0, 0);
1131                 scatterwalk_done(&assoc_sg_walk, 0, 0);
1132         } else {
1133                 scatterwalk_map_and_copy(dst, req->dst, 0,
1134                         req->cryptlen + auth_tag_len, 1);
1135                 kfree(src);
1136         }
1137         return 0;
1138 }
1139
1140 static int __driver_rfc4106_decrypt(struct aead_request *req)
1141 {
1142         u8 one_entry_in_sg = 0;
1143         u8 *src, *dst, *assoc;
1144         unsigned long tempCipherLen = 0;
1145         __be32 counter = cpu_to_be32(1);
1146         int retval = 0;
1147         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1148         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
1149         void *aes_ctx = &(ctx->aes_key_expanded);
1150         unsigned long auth_tag_len = crypto_aead_authsize(tfm);
1151         u8 iv_and_authTag[32+AESNI_ALIGN];
1152         u8 *iv = (u8 *) PTR_ALIGN((u8 *)iv_and_authTag, AESNI_ALIGN);
1153         u8 *authTag = iv + 16;
1154         struct scatter_walk src_sg_walk;
1155         struct scatter_walk assoc_sg_walk;
1156         struct scatter_walk dst_sg_walk;
1157         unsigned int i;
1158
1159         if (unlikely((req->cryptlen < auth_tag_len) ||
1160                 (req->assoclen != 8 && req->assoclen != 12)))
1161                 return -EINVAL;
1162         /* Assuming we are supporting rfc4106 64-bit extended */
1163         /* sequence numbers We need to have the AAD length */
1164         /* equal to 8 or 12 bytes */
1165
1166         tempCipherLen = (unsigned long)(req->cryptlen - auth_tag_len);
1167         /* IV below built */
1168         for (i = 0; i < 4; i++)
1169                 *(iv+i) = ctx->nonce[i];
1170         for (i = 0; i < 8; i++)
1171                 *(iv+4+i) = req->iv[i];
1172         *((__be32 *)(iv+12)) = counter;
1173
1174         if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) {
1175                 one_entry_in_sg = 1;
1176                 scatterwalk_start(&src_sg_walk, req->src);
1177                 scatterwalk_start(&assoc_sg_walk, req->assoc);
1178                 src = scatterwalk_map(&src_sg_walk, 0);
1179                 assoc = scatterwalk_map(&assoc_sg_walk, 0);
1180                 dst = src;
1181                 if (unlikely(req->src != req->dst)) {
1182                         scatterwalk_start(&dst_sg_walk, req->dst);
1183                         dst = scatterwalk_map(&dst_sg_walk, 0);
1184                 }
1185
1186         } else {
1187                 /* Allocate memory for src, dst, assoc */
1188                 src = kmalloc(req->cryptlen + req->assoclen, GFP_ATOMIC);
1189                 if (!src)
1190                         return -ENOMEM;
1191                 assoc = (src + req->cryptlen + auth_tag_len);
1192                 scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0);
1193                 scatterwalk_map_and_copy(assoc, req->assoc, 0,
1194                         req->assoclen, 0);
1195                 dst = src;
1196         }
1197
1198         aesni_gcm_dec(aes_ctx, dst, src, tempCipherLen, iv,
1199                 ctx->hash_subkey, assoc, (unsigned long)req->assoclen,
1200                 authTag, auth_tag_len);
1201
1202         /* Compare generated tag with passed in tag. */
1203         retval = memcmp(src + tempCipherLen, authTag, auth_tag_len) ?
1204                 -EBADMSG : 0;
1205
1206         if (one_entry_in_sg) {
1207                 if (unlikely(req->src != req->dst)) {
1208                         scatterwalk_unmap(dst, 0);
1209                         scatterwalk_done(&dst_sg_walk, 0, 0);
1210                 }
1211                 scatterwalk_unmap(src, 0);
1212                 scatterwalk_unmap(assoc, 0);
1213                 scatterwalk_done(&src_sg_walk, 0, 0);
1214                 scatterwalk_done(&assoc_sg_walk, 0, 0);
1215         } else {
1216                 scatterwalk_map_and_copy(dst, req->dst, 0, req->cryptlen, 1);
1217                 kfree(src);
1218         }
1219         return retval;
1220 }
1221
1222 static struct crypto_alg __rfc4106_alg = {
1223         .cra_name               = "__gcm-aes-aesni",
1224         .cra_driver_name        = "__driver-gcm-aes-aesni",
1225         .cra_priority           = 0,
1226         .cra_flags              = CRYPTO_ALG_TYPE_AEAD,
1227         .cra_blocksize          = 1,
1228         .cra_ctxsize    = sizeof(struct aesni_rfc4106_gcm_ctx) + AESNI_ALIGN,
1229         .cra_alignmask          = 0,
1230         .cra_type               = &crypto_aead_type,
1231         .cra_module             = THIS_MODULE,
1232         .cra_list               = LIST_HEAD_INIT(__rfc4106_alg.cra_list),
1233         .cra_u = {
1234                 .aead = {
1235                         .encrypt        = __driver_rfc4106_encrypt,
1236                         .decrypt        = __driver_rfc4106_decrypt,
1237                 },
1238         },
1239 };
1240
1241 static int __init aesni_init(void)
1242 {
1243         int err;
1244
1245         if (!cpu_has_aes) {
1246                 printk(KERN_INFO "Intel AES-NI instructions are not detected.\n");
1247                 return -ENODEV;
1248         }
1249
1250         if ((err = crypto_register_alg(&aesni_alg)))
1251                 goto aes_err;
1252         if ((err = crypto_register_alg(&__aesni_alg)))
1253                 goto __aes_err;
1254         if ((err = crypto_register_alg(&blk_ecb_alg)))
1255                 goto blk_ecb_err;
1256         if ((err = crypto_register_alg(&blk_cbc_alg)))
1257                 goto blk_cbc_err;
1258         if ((err = crypto_register_alg(&ablk_ecb_alg)))
1259                 goto ablk_ecb_err;
1260         if ((err = crypto_register_alg(&ablk_cbc_alg)))
1261                 goto ablk_cbc_err;
1262 #ifdef CONFIG_X86_64
1263         if ((err = crypto_register_alg(&blk_ctr_alg)))
1264                 goto blk_ctr_err;
1265         if ((err = crypto_register_alg(&ablk_ctr_alg)))
1266                 goto ablk_ctr_err;
1267 #ifdef HAS_CTR
1268         if ((err = crypto_register_alg(&ablk_rfc3686_ctr_alg)))
1269                 goto ablk_rfc3686_ctr_err;
1270 #endif
1271 #endif
1272 #ifdef HAS_LRW
1273         if ((err = crypto_register_alg(&ablk_lrw_alg)))
1274                 goto ablk_lrw_err;
1275 #endif
1276 #ifdef HAS_PCBC
1277         if ((err = crypto_register_alg(&ablk_pcbc_alg)))
1278                 goto ablk_pcbc_err;
1279 #endif
1280 #ifdef HAS_XTS
1281         if ((err = crypto_register_alg(&ablk_xts_alg)))
1282                 goto ablk_xts_err;
1283 #endif
1284         err = crypto_register_alg(&__rfc4106_alg);
1285         if (err)
1286                 goto __aead_gcm_err;
1287         err = crypto_register_alg(&rfc4106_alg);
1288         if (err)
1289                 goto aead_gcm_err;
1290         return err;
1291
1292 aead_gcm_err:
1293         crypto_unregister_alg(&__rfc4106_alg);
1294 __aead_gcm_err:
1295 #ifdef HAS_XTS
1296         crypto_unregister_alg(&ablk_xts_alg);
1297 ablk_xts_err:
1298 #endif
1299 #ifdef HAS_PCBC
1300         crypto_unregister_alg(&ablk_pcbc_alg);
1301 ablk_pcbc_err:
1302 #endif
1303 #ifdef HAS_LRW
1304         crypto_unregister_alg(&ablk_lrw_alg);
1305 ablk_lrw_err:
1306 #endif
1307 #ifdef CONFIG_X86_64
1308 #ifdef HAS_CTR
1309         crypto_unregister_alg(&ablk_rfc3686_ctr_alg);
1310 ablk_rfc3686_ctr_err:
1311 #endif
1312         crypto_unregister_alg(&ablk_ctr_alg);
1313 ablk_ctr_err:
1314         crypto_unregister_alg(&blk_ctr_alg);
1315 blk_ctr_err:
1316 #endif
1317         crypto_unregister_alg(&ablk_cbc_alg);
1318 ablk_cbc_err:
1319         crypto_unregister_alg(&ablk_ecb_alg);
1320 ablk_ecb_err:
1321         crypto_unregister_alg(&blk_cbc_alg);
1322 blk_cbc_err:
1323         crypto_unregister_alg(&blk_ecb_alg);
1324 blk_ecb_err:
1325         crypto_unregister_alg(&__aesni_alg);
1326 __aes_err:
1327         crypto_unregister_alg(&aesni_alg);
1328 aes_err:
1329         return err;
1330 }
1331
1332 static void __exit aesni_exit(void)
1333 {
1334         crypto_unregister_alg(&__rfc4106_alg);
1335         crypto_unregister_alg(&rfc4106_alg);
1336 #ifdef HAS_XTS
1337         crypto_unregister_alg(&ablk_xts_alg);
1338 #endif
1339 #ifdef HAS_PCBC
1340         crypto_unregister_alg(&ablk_pcbc_alg);
1341 #endif
1342 #ifdef HAS_LRW
1343         crypto_unregister_alg(&ablk_lrw_alg);
1344 #endif
1345 #ifdef CONFIG_X86_64
1346 #ifdef HAS_CTR
1347         crypto_unregister_alg(&ablk_rfc3686_ctr_alg);
1348 #endif
1349         crypto_unregister_alg(&ablk_ctr_alg);
1350         crypto_unregister_alg(&blk_ctr_alg);
1351 #endif
1352         crypto_unregister_alg(&ablk_cbc_alg);
1353         crypto_unregister_alg(&ablk_ecb_alg);
1354         crypto_unregister_alg(&blk_cbc_alg);
1355         crypto_unregister_alg(&blk_ecb_alg);
1356         crypto_unregister_alg(&__aesni_alg);
1357         crypto_unregister_alg(&aesni_alg);
1358 }
1359
1360 module_init(aesni_init);
1361 module_exit(aesni_exit);
1362
1363 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized");
1364 MODULE_LICENSE("GPL");
1365 MODULE_ALIAS("aes");