crypto: aesni-intel - Fixed build error on x86-32
[linux-2.6.git] / arch / x86 / crypto / aesni-intel_glue.c
1 /*
2  * Support for Intel AES-NI instructions. This file contains glue
3  * code, the real AES implementation is in intel-aes_asm.S.
4  *
5  * Copyright (C) 2008, Intel Corp.
6  *    Author: Huang Ying <ying.huang@intel.com>
7  *
8  * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
9  * interface for 64-bit kernels.
10  *    Authors: Adrian Hoban <adrian.hoban@intel.com>
11  *             Gabriele Paoloni <gabriele.paoloni@intel.com>
12  *             Tadeusz Struk (tadeusz.struk@intel.com)
13  *             Aidan O'Mahony (aidan.o.mahony@intel.com)
14  *    Copyright (c) 2010, Intel Corporation.
15  *
16  * This program is free software; you can redistribute it and/or modify
17  * it under the terms of the GNU General Public License as published by
18  * the Free Software Foundation; either version 2 of the License, or
19  * (at your option) any later version.
20  */
21
22 #include <linux/hardirq.h>
23 #include <linux/types.h>
24 #include <linux/crypto.h>
25 #include <linux/err.h>
26 #include <crypto/algapi.h>
27 #include <crypto/aes.h>
28 #include <crypto/cryptd.h>
29 #include <crypto/ctr.h>
30 #include <asm/i387.h>
31 #include <asm/aes.h>
32 #include <crypto/scatterwalk.h>
33 #include <crypto/internal/aead.h>
34 #include <linux/workqueue.h>
35 #include <linux/spinlock.h>
36
37 #if defined(CONFIG_CRYPTO_CTR) || defined(CONFIG_CRYPTO_CTR_MODULE)
38 #define HAS_CTR
39 #endif
40
41 #if defined(CONFIG_CRYPTO_LRW) || defined(CONFIG_CRYPTO_LRW_MODULE)
42 #define HAS_LRW
43 #endif
44
45 #if defined(CONFIG_CRYPTO_PCBC) || defined(CONFIG_CRYPTO_PCBC_MODULE)
46 #define HAS_PCBC
47 #endif
48
49 #if defined(CONFIG_CRYPTO_XTS) || defined(CONFIG_CRYPTO_XTS_MODULE)
50 #define HAS_XTS
51 #endif
52
53 struct async_aes_ctx {
54         struct cryptd_ablkcipher *cryptd_tfm;
55 };
56
57 /* This data is stored at the end of the crypto_tfm struct.
58  * It's a type of per "session" data storage location.
59  * This needs to be 16 byte aligned.
60  */
61 struct aesni_rfc4106_gcm_ctx {
62         u8 hash_subkey[16];
63         struct crypto_aes_ctx aes_key_expanded;
64         u8 nonce[4];
65         struct cryptd_aead *cryptd_tfm;
66 };
67
68 struct aesni_gcm_set_hash_subkey_result {
69         int err;
70         struct completion completion;
71 };
72
73 struct aesni_hash_subkey_req_data {
74         u8 iv[16];
75         struct aesni_gcm_set_hash_subkey_result result;
76         struct scatterlist sg;
77 };
78
79 #define AESNI_ALIGN     (16)
80 #define AES_BLOCK_MASK  (~(AES_BLOCK_SIZE-1))
81 #define RFC4106_HASH_SUBKEY_SIZE 16
82
83 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
84                              unsigned int key_len);
85 asmlinkage void aesni_enc(struct crypto_aes_ctx *ctx, u8 *out,
86                           const u8 *in);
87 asmlinkage void aesni_dec(struct crypto_aes_ctx *ctx, u8 *out,
88                           const u8 *in);
89 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out,
90                               const u8 *in, unsigned int len);
91 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out,
92                               const u8 *in, unsigned int len);
93 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
94                               const u8 *in, unsigned int len, u8 *iv);
95 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
96                               const u8 *in, unsigned int len, u8 *iv);
97 #ifdef CONFIG_X86_64
98 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out,
99                               const u8 *in, unsigned int len, u8 *iv);
100
101 /* asmlinkage void aesni_gcm_enc()
102  * void *ctx,  AES Key schedule. Starts on a 16 byte boundary.
103  * u8 *out, Ciphertext output. Encrypt in-place is allowed.
104  * const u8 *in, Plaintext input
105  * unsigned long plaintext_len, Length of data in bytes for encryption.
106  * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association)
107  *         concatenated with 8 byte Initialisation Vector (from IPSec ESP
108  *         Payload) concatenated with 0x00000001. 16-byte aligned pointer.
109  * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
110  * const u8 *aad, Additional Authentication Data (AAD)
111  * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this
112  *          is going to be 8 or 12 bytes
113  * u8 *auth_tag, Authenticated Tag output.
114  * unsigned long auth_tag_len), Authenticated Tag Length in bytes.
115  *          Valid values are 16 (most likely), 12 or 8.
116  */
117 asmlinkage void aesni_gcm_enc(void *ctx, u8 *out,
118                         const u8 *in, unsigned long plaintext_len, u8 *iv,
119                         u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
120                         u8 *auth_tag, unsigned long auth_tag_len);
121
122 /* asmlinkage void aesni_gcm_dec()
123  * void *ctx, AES Key schedule. Starts on a 16 byte boundary.
124  * u8 *out, Plaintext output. Decrypt in-place is allowed.
125  * const u8 *in, Ciphertext input
126  * unsigned long ciphertext_len, Length of data in bytes for decryption.
127  * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association)
128  *         concatenated with 8 byte Initialisation Vector (from IPSec ESP
129  *         Payload) concatenated with 0x00000001. 16-byte aligned pointer.
130  * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
131  * const u8 *aad, Additional Authentication Data (AAD)
132  * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this is going
133  * to be 8 or 12 bytes
134  * u8 *auth_tag, Authenticated Tag output.
135  * unsigned long auth_tag_len) Authenticated Tag Length in bytes.
136  * Valid values are 16 (most likely), 12 or 8.
137  */
138 asmlinkage void aesni_gcm_dec(void *ctx, u8 *out,
139                         const u8 *in, unsigned long ciphertext_len, u8 *iv,
140                         u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
141                         u8 *auth_tag, unsigned long auth_tag_len);
142
143 static inline struct
144 aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm)
145 {
146         return
147                 (struct aesni_rfc4106_gcm_ctx *)
148                 PTR_ALIGN((u8 *)
149                 crypto_tfm_ctx(crypto_aead_tfm(tfm)), AESNI_ALIGN);
150 }
151 #endif
152
153 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
154 {
155         unsigned long addr = (unsigned long)raw_ctx;
156         unsigned long align = AESNI_ALIGN;
157
158         if (align <= crypto_tfm_ctx_alignment())
159                 align = 1;
160         return (struct crypto_aes_ctx *)ALIGN(addr, align);
161 }
162
163 static int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx,
164                               const u8 *in_key, unsigned int key_len)
165 {
166         struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx);
167         u32 *flags = &tfm->crt_flags;
168         int err;
169
170         if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 &&
171             key_len != AES_KEYSIZE_256) {
172                 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
173                 return -EINVAL;
174         }
175
176         if (!irq_fpu_usable())
177                 err = crypto_aes_expand_key(ctx, in_key, key_len);
178         else {
179                 kernel_fpu_begin();
180                 err = aesni_set_key(ctx, in_key, key_len);
181                 kernel_fpu_end();
182         }
183
184         return err;
185 }
186
187 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
188                        unsigned int key_len)
189 {
190         return aes_set_key_common(tfm, crypto_tfm_ctx(tfm), in_key, key_len);
191 }
192
193 static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
194 {
195         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
196
197         if (!irq_fpu_usable())
198                 crypto_aes_encrypt_x86(ctx, dst, src);
199         else {
200                 kernel_fpu_begin();
201                 aesni_enc(ctx, dst, src);
202                 kernel_fpu_end();
203         }
204 }
205
206 static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
207 {
208         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
209
210         if (!irq_fpu_usable())
211                 crypto_aes_decrypt_x86(ctx, dst, src);
212         else {
213                 kernel_fpu_begin();
214                 aesni_dec(ctx, dst, src);
215                 kernel_fpu_end();
216         }
217 }
218
219 static struct crypto_alg aesni_alg = {
220         .cra_name               = "aes",
221         .cra_driver_name        = "aes-aesni",
222         .cra_priority           = 300,
223         .cra_flags              = CRYPTO_ALG_TYPE_CIPHER,
224         .cra_blocksize          = AES_BLOCK_SIZE,
225         .cra_ctxsize            = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
226         .cra_alignmask          = 0,
227         .cra_module             = THIS_MODULE,
228         .cra_list               = LIST_HEAD_INIT(aesni_alg.cra_list),
229         .cra_u  = {
230                 .cipher = {
231                         .cia_min_keysize        = AES_MIN_KEY_SIZE,
232                         .cia_max_keysize        = AES_MAX_KEY_SIZE,
233                         .cia_setkey             = aes_set_key,
234                         .cia_encrypt            = aes_encrypt,
235                         .cia_decrypt            = aes_decrypt
236                 }
237         }
238 };
239
240 static void __aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
241 {
242         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
243
244         aesni_enc(ctx, dst, src);
245 }
246
247 static void __aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
248 {
249         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
250
251         aesni_dec(ctx, dst, src);
252 }
253
254 static struct crypto_alg __aesni_alg = {
255         .cra_name               = "__aes-aesni",
256         .cra_driver_name        = "__driver-aes-aesni",
257         .cra_priority           = 0,
258         .cra_flags              = CRYPTO_ALG_TYPE_CIPHER,
259         .cra_blocksize          = AES_BLOCK_SIZE,
260         .cra_ctxsize            = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
261         .cra_alignmask          = 0,
262         .cra_module             = THIS_MODULE,
263         .cra_list               = LIST_HEAD_INIT(__aesni_alg.cra_list),
264         .cra_u  = {
265                 .cipher = {
266                         .cia_min_keysize        = AES_MIN_KEY_SIZE,
267                         .cia_max_keysize        = AES_MAX_KEY_SIZE,
268                         .cia_setkey             = aes_set_key,
269                         .cia_encrypt            = __aes_encrypt,
270                         .cia_decrypt            = __aes_decrypt
271                 }
272         }
273 };
274
275 static int ecb_encrypt(struct blkcipher_desc *desc,
276                        struct scatterlist *dst, struct scatterlist *src,
277                        unsigned int nbytes)
278 {
279         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
280         struct blkcipher_walk walk;
281         int err;
282
283         blkcipher_walk_init(&walk, dst, src, nbytes);
284         err = blkcipher_walk_virt(desc, &walk);
285         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
286
287         kernel_fpu_begin();
288         while ((nbytes = walk.nbytes)) {
289                 aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
290                               nbytes & AES_BLOCK_MASK);
291                 nbytes &= AES_BLOCK_SIZE - 1;
292                 err = blkcipher_walk_done(desc, &walk, nbytes);
293         }
294         kernel_fpu_end();
295
296         return err;
297 }
298
299 static int ecb_decrypt(struct blkcipher_desc *desc,
300                        struct scatterlist *dst, struct scatterlist *src,
301                        unsigned int nbytes)
302 {
303         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
304         struct blkcipher_walk walk;
305         int err;
306
307         blkcipher_walk_init(&walk, dst, src, nbytes);
308         err = blkcipher_walk_virt(desc, &walk);
309         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
310
311         kernel_fpu_begin();
312         while ((nbytes = walk.nbytes)) {
313                 aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
314                               nbytes & AES_BLOCK_MASK);
315                 nbytes &= AES_BLOCK_SIZE - 1;
316                 err = blkcipher_walk_done(desc, &walk, nbytes);
317         }
318         kernel_fpu_end();
319
320         return err;
321 }
322
323 static struct crypto_alg blk_ecb_alg = {
324         .cra_name               = "__ecb-aes-aesni",
325         .cra_driver_name        = "__driver-ecb-aes-aesni",
326         .cra_priority           = 0,
327         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
328         .cra_blocksize          = AES_BLOCK_SIZE,
329         .cra_ctxsize            = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
330         .cra_alignmask          = 0,
331         .cra_type               = &crypto_blkcipher_type,
332         .cra_module             = THIS_MODULE,
333         .cra_list               = LIST_HEAD_INIT(blk_ecb_alg.cra_list),
334         .cra_u = {
335                 .blkcipher = {
336                         .min_keysize    = AES_MIN_KEY_SIZE,
337                         .max_keysize    = AES_MAX_KEY_SIZE,
338                         .setkey         = aes_set_key,
339                         .encrypt        = ecb_encrypt,
340                         .decrypt        = ecb_decrypt,
341                 },
342         },
343 };
344
345 static int cbc_encrypt(struct blkcipher_desc *desc,
346                        struct scatterlist *dst, struct scatterlist *src,
347                        unsigned int nbytes)
348 {
349         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
350         struct blkcipher_walk walk;
351         int err;
352
353         blkcipher_walk_init(&walk, dst, src, nbytes);
354         err = blkcipher_walk_virt(desc, &walk);
355         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
356
357         kernel_fpu_begin();
358         while ((nbytes = walk.nbytes)) {
359                 aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
360                               nbytes & AES_BLOCK_MASK, walk.iv);
361                 nbytes &= AES_BLOCK_SIZE - 1;
362                 err = blkcipher_walk_done(desc, &walk, nbytes);
363         }
364         kernel_fpu_end();
365
366         return err;
367 }
368
369 static int cbc_decrypt(struct blkcipher_desc *desc,
370                        struct scatterlist *dst, struct scatterlist *src,
371                        unsigned int nbytes)
372 {
373         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
374         struct blkcipher_walk walk;
375         int err;
376
377         blkcipher_walk_init(&walk, dst, src, nbytes);
378         err = blkcipher_walk_virt(desc, &walk);
379         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
380
381         kernel_fpu_begin();
382         while ((nbytes = walk.nbytes)) {
383                 aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
384                               nbytes & AES_BLOCK_MASK, walk.iv);
385                 nbytes &= AES_BLOCK_SIZE - 1;
386                 err = blkcipher_walk_done(desc, &walk, nbytes);
387         }
388         kernel_fpu_end();
389
390         return err;
391 }
392
393 static struct crypto_alg blk_cbc_alg = {
394         .cra_name               = "__cbc-aes-aesni",
395         .cra_driver_name        = "__driver-cbc-aes-aesni",
396         .cra_priority           = 0,
397         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
398         .cra_blocksize          = AES_BLOCK_SIZE,
399         .cra_ctxsize            = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
400         .cra_alignmask          = 0,
401         .cra_type               = &crypto_blkcipher_type,
402         .cra_module             = THIS_MODULE,
403         .cra_list               = LIST_HEAD_INIT(blk_cbc_alg.cra_list),
404         .cra_u = {
405                 .blkcipher = {
406                         .min_keysize    = AES_MIN_KEY_SIZE,
407                         .max_keysize    = AES_MAX_KEY_SIZE,
408                         .setkey         = aes_set_key,
409                         .encrypt        = cbc_encrypt,
410                         .decrypt        = cbc_decrypt,
411                 },
412         },
413 };
414
415 #ifdef CONFIG_X86_64
416 static void ctr_crypt_final(struct crypto_aes_ctx *ctx,
417                             struct blkcipher_walk *walk)
418 {
419         u8 *ctrblk = walk->iv;
420         u8 keystream[AES_BLOCK_SIZE];
421         u8 *src = walk->src.virt.addr;
422         u8 *dst = walk->dst.virt.addr;
423         unsigned int nbytes = walk->nbytes;
424
425         aesni_enc(ctx, keystream, ctrblk);
426         crypto_xor(keystream, src, nbytes);
427         memcpy(dst, keystream, nbytes);
428         crypto_inc(ctrblk, AES_BLOCK_SIZE);
429 }
430
431 static int ctr_crypt(struct blkcipher_desc *desc,
432                      struct scatterlist *dst, struct scatterlist *src,
433                      unsigned int nbytes)
434 {
435         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
436         struct blkcipher_walk walk;
437         int err;
438
439         blkcipher_walk_init(&walk, dst, src, nbytes);
440         err = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE);
441         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
442
443         kernel_fpu_begin();
444         while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
445                 aesni_ctr_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
446                               nbytes & AES_BLOCK_MASK, walk.iv);
447                 nbytes &= AES_BLOCK_SIZE - 1;
448                 err = blkcipher_walk_done(desc, &walk, nbytes);
449         }
450         if (walk.nbytes) {
451                 ctr_crypt_final(ctx, &walk);
452                 err = blkcipher_walk_done(desc, &walk, 0);
453         }
454         kernel_fpu_end();
455
456         return err;
457 }
458
459 static struct crypto_alg blk_ctr_alg = {
460         .cra_name               = "__ctr-aes-aesni",
461         .cra_driver_name        = "__driver-ctr-aes-aesni",
462         .cra_priority           = 0,
463         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
464         .cra_blocksize          = 1,
465         .cra_ctxsize            = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
466         .cra_alignmask          = 0,
467         .cra_type               = &crypto_blkcipher_type,
468         .cra_module             = THIS_MODULE,
469         .cra_list               = LIST_HEAD_INIT(blk_ctr_alg.cra_list),
470         .cra_u = {
471                 .blkcipher = {
472                         .min_keysize    = AES_MIN_KEY_SIZE,
473                         .max_keysize    = AES_MAX_KEY_SIZE,
474                         .ivsize         = AES_BLOCK_SIZE,
475                         .setkey         = aes_set_key,
476                         .encrypt        = ctr_crypt,
477                         .decrypt        = ctr_crypt,
478                 },
479         },
480 };
481 #endif
482
483 static int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key,
484                         unsigned int key_len)
485 {
486         struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
487         struct crypto_ablkcipher *child = &ctx->cryptd_tfm->base;
488         int err;
489
490         crypto_ablkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
491         crypto_ablkcipher_set_flags(child, crypto_ablkcipher_get_flags(tfm)
492                                     & CRYPTO_TFM_REQ_MASK);
493         err = crypto_ablkcipher_setkey(child, key, key_len);
494         crypto_ablkcipher_set_flags(tfm, crypto_ablkcipher_get_flags(child)
495                                     & CRYPTO_TFM_RES_MASK);
496         return err;
497 }
498
499 static int ablk_encrypt(struct ablkcipher_request *req)
500 {
501         struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
502         struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
503
504         if (!irq_fpu_usable()) {
505                 struct ablkcipher_request *cryptd_req =
506                         ablkcipher_request_ctx(req);
507                 memcpy(cryptd_req, req, sizeof(*req));
508                 ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
509                 return crypto_ablkcipher_encrypt(cryptd_req);
510         } else {
511                 struct blkcipher_desc desc;
512                 desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm);
513                 desc.info = req->info;
514                 desc.flags = 0;
515                 return crypto_blkcipher_crt(desc.tfm)->encrypt(
516                         &desc, req->dst, req->src, req->nbytes);
517         }
518 }
519
520 static int ablk_decrypt(struct ablkcipher_request *req)
521 {
522         struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
523         struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
524
525         if (!irq_fpu_usable()) {
526                 struct ablkcipher_request *cryptd_req =
527                         ablkcipher_request_ctx(req);
528                 memcpy(cryptd_req, req, sizeof(*req));
529                 ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
530                 return crypto_ablkcipher_decrypt(cryptd_req);
531         } else {
532                 struct blkcipher_desc desc;
533                 desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm);
534                 desc.info = req->info;
535                 desc.flags = 0;
536                 return crypto_blkcipher_crt(desc.tfm)->decrypt(
537                         &desc, req->dst, req->src, req->nbytes);
538         }
539 }
540
541 static void ablk_exit(struct crypto_tfm *tfm)
542 {
543         struct async_aes_ctx *ctx = crypto_tfm_ctx(tfm);
544
545         cryptd_free_ablkcipher(ctx->cryptd_tfm);
546 }
547
548 static void ablk_init_common(struct crypto_tfm *tfm,
549                              struct cryptd_ablkcipher *cryptd_tfm)
550 {
551         struct async_aes_ctx *ctx = crypto_tfm_ctx(tfm);
552
553         ctx->cryptd_tfm = cryptd_tfm;
554         tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request) +
555                 crypto_ablkcipher_reqsize(&cryptd_tfm->base);
556 }
557
558 static int ablk_ecb_init(struct crypto_tfm *tfm)
559 {
560         struct cryptd_ablkcipher *cryptd_tfm;
561
562         cryptd_tfm = cryptd_alloc_ablkcipher("__driver-ecb-aes-aesni", 0, 0);
563         if (IS_ERR(cryptd_tfm))
564                 return PTR_ERR(cryptd_tfm);
565         ablk_init_common(tfm, cryptd_tfm);
566         return 0;
567 }
568
569 static struct crypto_alg ablk_ecb_alg = {
570         .cra_name               = "ecb(aes)",
571         .cra_driver_name        = "ecb-aes-aesni",
572         .cra_priority           = 400,
573         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
574         .cra_blocksize          = AES_BLOCK_SIZE,
575         .cra_ctxsize            = sizeof(struct async_aes_ctx),
576         .cra_alignmask          = 0,
577         .cra_type               = &crypto_ablkcipher_type,
578         .cra_module             = THIS_MODULE,
579         .cra_list               = LIST_HEAD_INIT(ablk_ecb_alg.cra_list),
580         .cra_init               = ablk_ecb_init,
581         .cra_exit               = ablk_exit,
582         .cra_u = {
583                 .ablkcipher = {
584                         .min_keysize    = AES_MIN_KEY_SIZE,
585                         .max_keysize    = AES_MAX_KEY_SIZE,
586                         .setkey         = ablk_set_key,
587                         .encrypt        = ablk_encrypt,
588                         .decrypt        = ablk_decrypt,
589                 },
590         },
591 };
592
593 static int ablk_cbc_init(struct crypto_tfm *tfm)
594 {
595         struct cryptd_ablkcipher *cryptd_tfm;
596
597         cryptd_tfm = cryptd_alloc_ablkcipher("__driver-cbc-aes-aesni", 0, 0);
598         if (IS_ERR(cryptd_tfm))
599                 return PTR_ERR(cryptd_tfm);
600         ablk_init_common(tfm, cryptd_tfm);
601         return 0;
602 }
603
604 static struct crypto_alg ablk_cbc_alg = {
605         .cra_name               = "cbc(aes)",
606         .cra_driver_name        = "cbc-aes-aesni",
607         .cra_priority           = 400,
608         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
609         .cra_blocksize          = AES_BLOCK_SIZE,
610         .cra_ctxsize            = sizeof(struct async_aes_ctx),
611         .cra_alignmask          = 0,
612         .cra_type               = &crypto_ablkcipher_type,
613         .cra_module             = THIS_MODULE,
614         .cra_list               = LIST_HEAD_INIT(ablk_cbc_alg.cra_list),
615         .cra_init               = ablk_cbc_init,
616         .cra_exit               = ablk_exit,
617         .cra_u = {
618                 .ablkcipher = {
619                         .min_keysize    = AES_MIN_KEY_SIZE,
620                         .max_keysize    = AES_MAX_KEY_SIZE,
621                         .ivsize         = AES_BLOCK_SIZE,
622                         .setkey         = ablk_set_key,
623                         .encrypt        = ablk_encrypt,
624                         .decrypt        = ablk_decrypt,
625                 },
626         },
627 };
628
629 #ifdef CONFIG_X86_64
630 static int ablk_ctr_init(struct crypto_tfm *tfm)
631 {
632         struct cryptd_ablkcipher *cryptd_tfm;
633
634         cryptd_tfm = cryptd_alloc_ablkcipher("__driver-ctr-aes-aesni", 0, 0);
635         if (IS_ERR(cryptd_tfm))
636                 return PTR_ERR(cryptd_tfm);
637         ablk_init_common(tfm, cryptd_tfm);
638         return 0;
639 }
640
641 static struct crypto_alg ablk_ctr_alg = {
642         .cra_name               = "ctr(aes)",
643         .cra_driver_name        = "ctr-aes-aesni",
644         .cra_priority           = 400,
645         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
646         .cra_blocksize          = 1,
647         .cra_ctxsize            = sizeof(struct async_aes_ctx),
648         .cra_alignmask          = 0,
649         .cra_type               = &crypto_ablkcipher_type,
650         .cra_module             = THIS_MODULE,
651         .cra_list               = LIST_HEAD_INIT(ablk_ctr_alg.cra_list),
652         .cra_init               = ablk_ctr_init,
653         .cra_exit               = ablk_exit,
654         .cra_u = {
655                 .ablkcipher = {
656                         .min_keysize    = AES_MIN_KEY_SIZE,
657                         .max_keysize    = AES_MAX_KEY_SIZE,
658                         .ivsize         = AES_BLOCK_SIZE,
659                         .setkey         = ablk_set_key,
660                         .encrypt        = ablk_encrypt,
661                         .decrypt        = ablk_encrypt,
662                         .geniv          = "chainiv",
663                 },
664         },
665 };
666
667 #ifdef HAS_CTR
668 static int ablk_rfc3686_ctr_init(struct crypto_tfm *tfm)
669 {
670         struct cryptd_ablkcipher *cryptd_tfm;
671
672         cryptd_tfm = cryptd_alloc_ablkcipher(
673                 "rfc3686(__driver-ctr-aes-aesni)", 0, 0);
674         if (IS_ERR(cryptd_tfm))
675                 return PTR_ERR(cryptd_tfm);
676         ablk_init_common(tfm, cryptd_tfm);
677         return 0;
678 }
679
680 static struct crypto_alg ablk_rfc3686_ctr_alg = {
681         .cra_name               = "rfc3686(ctr(aes))",
682         .cra_driver_name        = "rfc3686-ctr-aes-aesni",
683         .cra_priority           = 400,
684         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
685         .cra_blocksize          = 1,
686         .cra_ctxsize            = sizeof(struct async_aes_ctx),
687         .cra_alignmask          = 0,
688         .cra_type               = &crypto_ablkcipher_type,
689         .cra_module             = THIS_MODULE,
690         .cra_list               = LIST_HEAD_INIT(ablk_rfc3686_ctr_alg.cra_list),
691         .cra_init               = ablk_rfc3686_ctr_init,
692         .cra_exit               = ablk_exit,
693         .cra_u = {
694                 .ablkcipher = {
695                         .min_keysize = AES_MIN_KEY_SIZE+CTR_RFC3686_NONCE_SIZE,
696                         .max_keysize = AES_MAX_KEY_SIZE+CTR_RFC3686_NONCE_SIZE,
697                         .ivsize      = CTR_RFC3686_IV_SIZE,
698                         .setkey      = ablk_set_key,
699                         .encrypt     = ablk_encrypt,
700                         .decrypt     = ablk_decrypt,
701                         .geniv       = "seqiv",
702                 },
703         },
704 };
705 #endif
706 #endif
707
708 #ifdef HAS_LRW
709 static int ablk_lrw_init(struct crypto_tfm *tfm)
710 {
711         struct cryptd_ablkcipher *cryptd_tfm;
712
713         cryptd_tfm = cryptd_alloc_ablkcipher("fpu(lrw(__driver-aes-aesni))",
714                                              0, 0);
715         if (IS_ERR(cryptd_tfm))
716                 return PTR_ERR(cryptd_tfm);
717         ablk_init_common(tfm, cryptd_tfm);
718         return 0;
719 }
720
721 static struct crypto_alg ablk_lrw_alg = {
722         .cra_name               = "lrw(aes)",
723         .cra_driver_name        = "lrw-aes-aesni",
724         .cra_priority           = 400,
725         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
726         .cra_blocksize          = AES_BLOCK_SIZE,
727         .cra_ctxsize            = sizeof(struct async_aes_ctx),
728         .cra_alignmask          = 0,
729         .cra_type               = &crypto_ablkcipher_type,
730         .cra_module             = THIS_MODULE,
731         .cra_list               = LIST_HEAD_INIT(ablk_lrw_alg.cra_list),
732         .cra_init               = ablk_lrw_init,
733         .cra_exit               = ablk_exit,
734         .cra_u = {
735                 .ablkcipher = {
736                         .min_keysize    = AES_MIN_KEY_SIZE + AES_BLOCK_SIZE,
737                         .max_keysize    = AES_MAX_KEY_SIZE + AES_BLOCK_SIZE,
738                         .ivsize         = AES_BLOCK_SIZE,
739                         .setkey         = ablk_set_key,
740                         .encrypt        = ablk_encrypt,
741                         .decrypt        = ablk_decrypt,
742                 },
743         },
744 };
745 #endif
746
747 #ifdef HAS_PCBC
748 static int ablk_pcbc_init(struct crypto_tfm *tfm)
749 {
750         struct cryptd_ablkcipher *cryptd_tfm;
751
752         cryptd_tfm = cryptd_alloc_ablkcipher("fpu(pcbc(__driver-aes-aesni))",
753                                              0, 0);
754         if (IS_ERR(cryptd_tfm))
755                 return PTR_ERR(cryptd_tfm);
756         ablk_init_common(tfm, cryptd_tfm);
757         return 0;
758 }
759
760 static struct crypto_alg ablk_pcbc_alg = {
761         .cra_name               = "pcbc(aes)",
762         .cra_driver_name        = "pcbc-aes-aesni",
763         .cra_priority           = 400,
764         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
765         .cra_blocksize          = AES_BLOCK_SIZE,
766         .cra_ctxsize            = sizeof(struct async_aes_ctx),
767         .cra_alignmask          = 0,
768         .cra_type               = &crypto_ablkcipher_type,
769         .cra_module             = THIS_MODULE,
770         .cra_list               = LIST_HEAD_INIT(ablk_pcbc_alg.cra_list),
771         .cra_init               = ablk_pcbc_init,
772         .cra_exit               = ablk_exit,
773         .cra_u = {
774                 .ablkcipher = {
775                         .min_keysize    = AES_MIN_KEY_SIZE,
776                         .max_keysize    = AES_MAX_KEY_SIZE,
777                         .ivsize         = AES_BLOCK_SIZE,
778                         .setkey         = ablk_set_key,
779                         .encrypt        = ablk_encrypt,
780                         .decrypt        = ablk_decrypt,
781                 },
782         },
783 };
784 #endif
785
786 #ifdef HAS_XTS
787 static int ablk_xts_init(struct crypto_tfm *tfm)
788 {
789         struct cryptd_ablkcipher *cryptd_tfm;
790
791         cryptd_tfm = cryptd_alloc_ablkcipher("fpu(xts(__driver-aes-aesni))",
792                                              0, 0);
793         if (IS_ERR(cryptd_tfm))
794                 return PTR_ERR(cryptd_tfm);
795         ablk_init_common(tfm, cryptd_tfm);
796         return 0;
797 }
798
799 static struct crypto_alg ablk_xts_alg = {
800         .cra_name               = "xts(aes)",
801         .cra_driver_name        = "xts-aes-aesni",
802         .cra_priority           = 400,
803         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
804         .cra_blocksize          = AES_BLOCK_SIZE,
805         .cra_ctxsize            = sizeof(struct async_aes_ctx),
806         .cra_alignmask          = 0,
807         .cra_type               = &crypto_ablkcipher_type,
808         .cra_module             = THIS_MODULE,
809         .cra_list               = LIST_HEAD_INIT(ablk_xts_alg.cra_list),
810         .cra_init               = ablk_xts_init,
811         .cra_exit               = ablk_exit,
812         .cra_u = {
813                 .ablkcipher = {
814                         .min_keysize    = 2 * AES_MIN_KEY_SIZE,
815                         .max_keysize    = 2 * AES_MAX_KEY_SIZE,
816                         .ivsize         = AES_BLOCK_SIZE,
817                         .setkey         = ablk_set_key,
818                         .encrypt        = ablk_encrypt,
819                         .decrypt        = ablk_decrypt,
820                 },
821         },
822 };
823 #endif
824
825 #ifdef CONFIG_X86_64
826 static int rfc4106_init(struct crypto_tfm *tfm)
827 {
828         struct cryptd_aead *cryptd_tfm;
829         struct aesni_rfc4106_gcm_ctx *ctx = (struct aesni_rfc4106_gcm_ctx *)
830                 PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN);
831         cryptd_tfm = cryptd_alloc_aead("__driver-gcm-aes-aesni", 0, 0);
832         if (IS_ERR(cryptd_tfm))
833                 return PTR_ERR(cryptd_tfm);
834         ctx->cryptd_tfm = cryptd_tfm;
835         tfm->crt_aead.reqsize = sizeof(struct aead_request)
836                 + crypto_aead_reqsize(&cryptd_tfm->base);
837         return 0;
838 }
839
840 static void rfc4106_exit(struct crypto_tfm *tfm)
841 {
842         struct aesni_rfc4106_gcm_ctx *ctx =
843                 (struct aesni_rfc4106_gcm_ctx *)
844                 PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN);
845         if (!IS_ERR(ctx->cryptd_tfm))
846                 cryptd_free_aead(ctx->cryptd_tfm);
847         return;
848 }
849
850 static void
851 rfc4106_set_hash_subkey_done(struct crypto_async_request *req, int err)
852 {
853         struct aesni_gcm_set_hash_subkey_result *result = req->data;
854
855         if (err == -EINPROGRESS)
856                 return;
857         result->err = err;
858         complete(&result->completion);
859 }
860
861 static int
862 rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len)
863 {
864         struct crypto_ablkcipher *ctr_tfm;
865         struct ablkcipher_request *req;
866         int ret = -EINVAL;
867         struct aesni_hash_subkey_req_data *req_data;
868
869         ctr_tfm = crypto_alloc_ablkcipher("ctr(aes)", 0, 0);
870         if (IS_ERR(ctr_tfm))
871                 return PTR_ERR(ctr_tfm);
872
873         crypto_ablkcipher_clear_flags(ctr_tfm, ~0);
874
875         ret = crypto_ablkcipher_setkey(ctr_tfm, key, key_len);
876         if (ret) {
877                 crypto_free_ablkcipher(ctr_tfm);
878                 return ret;
879         }
880
881         req = ablkcipher_request_alloc(ctr_tfm, GFP_KERNEL);
882         if (!req) {
883                 crypto_free_ablkcipher(ctr_tfm);
884                 return -EINVAL;
885         }
886
887         req_data = kmalloc(sizeof(*req_data), GFP_KERNEL);
888         if (!req_data) {
889                 crypto_free_ablkcipher(ctr_tfm);
890                 return -ENOMEM;
891         }
892         memset(req_data->iv, 0, sizeof(req_data->iv));
893
894         /* Clear the data in the hash sub key container to zero.*/
895         /* We want to cipher all zeros to create the hash sub key. */
896         memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE);
897
898         init_completion(&req_data->result.completion);
899         sg_init_one(&req_data->sg, hash_subkey, RFC4106_HASH_SUBKEY_SIZE);
900         ablkcipher_request_set_tfm(req, ctr_tfm);
901         ablkcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP |
902                                         CRYPTO_TFM_REQ_MAY_BACKLOG,
903                                         rfc4106_set_hash_subkey_done,
904                                         &req_data->result);
905
906         ablkcipher_request_set_crypt(req, &req_data->sg,
907                 &req_data->sg, RFC4106_HASH_SUBKEY_SIZE, req_data->iv);
908
909         ret = crypto_ablkcipher_encrypt(req);
910         if (ret == -EINPROGRESS || ret == -EBUSY) {
911                 ret = wait_for_completion_interruptible
912                         (&req_data->result.completion);
913                 if (!ret)
914                         ret = req_data->result.err;
915         }
916         ablkcipher_request_free(req);
917         kfree(req_data);
918         crypto_free_ablkcipher(ctr_tfm);
919         return ret;
920 }
921
922 static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key,
923                                                    unsigned int key_len)
924 {
925         int ret = 0;
926         struct crypto_tfm *tfm = crypto_aead_tfm(parent);
927         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent);
928         u8 *new_key_mem = NULL;
929
930         if (key_len < 4) {
931                 crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
932                 return -EINVAL;
933         }
934         /*Account for 4 byte nonce at the end.*/
935         key_len -= 4;
936         if (key_len != AES_KEYSIZE_128) {
937                 crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
938                 return -EINVAL;
939         }
940
941         memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce));
942         /*This must be on a 16 byte boundary!*/
943         if ((unsigned long)(&(ctx->aes_key_expanded.key_enc[0])) % AESNI_ALIGN)
944                 return -EINVAL;
945
946         if ((unsigned long)key % AESNI_ALIGN) {
947                 /*key is not aligned: use an auxuliar aligned pointer*/
948                 new_key_mem = kmalloc(key_len+AESNI_ALIGN, GFP_KERNEL);
949                 if (!new_key_mem)
950                         return -ENOMEM;
951
952                 new_key_mem = PTR_ALIGN(new_key_mem, AESNI_ALIGN);
953                 memcpy(new_key_mem, key, key_len);
954                 key = new_key_mem;
955         }
956
957         if (!irq_fpu_usable())
958                 ret = crypto_aes_expand_key(&(ctx->aes_key_expanded),
959                 key, key_len);
960         else {
961                 kernel_fpu_begin();
962                 ret = aesni_set_key(&(ctx->aes_key_expanded), key, key_len);
963                 kernel_fpu_end();
964         }
965         /*This must be on a 16 byte boundary!*/
966         if ((unsigned long)(&(ctx->hash_subkey[0])) % AESNI_ALIGN) {
967                 ret = -EINVAL;
968                 goto exit;
969         }
970         ret = rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
971 exit:
972         kfree(new_key_mem);
973         return ret;
974 }
975
976 /* This is the Integrity Check Value (aka the authentication tag length and can
977  * be 8, 12 or 16 bytes long. */
978 static int rfc4106_set_authsize(struct crypto_aead *parent,
979                                 unsigned int authsize)
980 {
981         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent);
982         struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
983
984         switch (authsize) {
985         case 8:
986         case 12:
987         case 16:
988                 break;
989         default:
990                 return -EINVAL;
991         }
992         crypto_aead_crt(parent)->authsize = authsize;
993         crypto_aead_crt(cryptd_child)->authsize = authsize;
994         return 0;
995 }
996
997 static int rfc4106_encrypt(struct aead_request *req)
998 {
999         int ret;
1000         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1001         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
1002         struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
1003
1004         if (!irq_fpu_usable()) {
1005                 struct aead_request *cryptd_req =
1006                         (struct aead_request *) aead_request_ctx(req);
1007                 memcpy(cryptd_req, req, sizeof(*req));
1008                 aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
1009                 return crypto_aead_encrypt(cryptd_req);
1010         } else {
1011                 kernel_fpu_begin();
1012                 ret = cryptd_child->base.crt_aead.encrypt(req);
1013                 kernel_fpu_end();
1014                 return ret;
1015         }
1016 }
1017
1018 static int rfc4106_decrypt(struct aead_request *req)
1019 {
1020         int ret;
1021         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1022         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
1023         struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
1024
1025         if (!irq_fpu_usable()) {
1026                 struct aead_request *cryptd_req =
1027                         (struct aead_request *) aead_request_ctx(req);
1028                 memcpy(cryptd_req, req, sizeof(*req));
1029                 aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
1030                 return crypto_aead_decrypt(cryptd_req);
1031         } else {
1032                 kernel_fpu_begin();
1033                 ret = cryptd_child->base.crt_aead.decrypt(req);
1034                 kernel_fpu_end();
1035                 return ret;
1036         }
1037 }
1038
1039 static struct crypto_alg rfc4106_alg = {
1040         .cra_name = "rfc4106(gcm(aes))",
1041         .cra_driver_name = "rfc4106-gcm-aesni",
1042         .cra_priority = 400,
1043         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
1044         .cra_blocksize = 1,
1045         .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx) + AESNI_ALIGN,
1046         .cra_alignmask = 0,
1047         .cra_type = &crypto_nivaead_type,
1048         .cra_module = THIS_MODULE,
1049         .cra_list = LIST_HEAD_INIT(rfc4106_alg.cra_list),
1050         .cra_init = rfc4106_init,
1051         .cra_exit = rfc4106_exit,
1052         .cra_u = {
1053                 .aead = {
1054                         .setkey = rfc4106_set_key,
1055                         .setauthsize = rfc4106_set_authsize,
1056                         .encrypt = rfc4106_encrypt,
1057                         .decrypt = rfc4106_decrypt,
1058                         .geniv = "seqiv",
1059                         .ivsize = 8,
1060                         .maxauthsize = 16,
1061                 },
1062         },
1063 };
1064
1065 static int __driver_rfc4106_encrypt(struct aead_request *req)
1066 {
1067         u8 one_entry_in_sg = 0;
1068         u8 *src, *dst, *assoc;
1069         __be32 counter = cpu_to_be32(1);
1070         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1071         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
1072         void *aes_ctx = &(ctx->aes_key_expanded);
1073         unsigned long auth_tag_len = crypto_aead_authsize(tfm);
1074         u8 iv_tab[16+AESNI_ALIGN];
1075         u8* iv = (u8 *) PTR_ALIGN((u8 *)iv_tab, AESNI_ALIGN);
1076         struct scatter_walk src_sg_walk;
1077         struct scatter_walk assoc_sg_walk;
1078         struct scatter_walk dst_sg_walk;
1079         unsigned int i;
1080
1081         /* Assuming we are supporting rfc4106 64-bit extended */
1082         /* sequence numbers We need to have the AAD length equal */
1083         /* to 8 or 12 bytes */
1084         if (unlikely(req->assoclen != 8 && req->assoclen != 12))
1085                 return -EINVAL;
1086         /* IV below built */
1087         for (i = 0; i < 4; i++)
1088                 *(iv+i) = ctx->nonce[i];
1089         for (i = 0; i < 8; i++)
1090                 *(iv+4+i) = req->iv[i];
1091         *((__be32 *)(iv+12)) = counter;
1092
1093         if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) {
1094                 one_entry_in_sg = 1;
1095                 scatterwalk_start(&src_sg_walk, req->src);
1096                 scatterwalk_start(&assoc_sg_walk, req->assoc);
1097                 src = scatterwalk_map(&src_sg_walk, 0);
1098                 assoc = scatterwalk_map(&assoc_sg_walk, 0);
1099                 dst = src;
1100                 if (unlikely(req->src != req->dst)) {
1101                         scatterwalk_start(&dst_sg_walk, req->dst);
1102                         dst = scatterwalk_map(&dst_sg_walk, 0);
1103                 }
1104
1105         } else {
1106                 /* Allocate memory for src, dst, assoc */
1107                 src = kmalloc(req->cryptlen + auth_tag_len + req->assoclen,
1108                         GFP_ATOMIC);
1109                 if (unlikely(!src))
1110                         return -ENOMEM;
1111                 assoc = (src + req->cryptlen + auth_tag_len);
1112                 scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0);
1113                 scatterwalk_map_and_copy(assoc, req->assoc, 0,
1114                                         req->assoclen, 0);
1115                 dst = src;
1116         }
1117
1118         aesni_gcm_enc(aes_ctx, dst, src, (unsigned long)req->cryptlen, iv,
1119                 ctx->hash_subkey, assoc, (unsigned long)req->assoclen, dst
1120                 + ((unsigned long)req->cryptlen), auth_tag_len);
1121
1122         /* The authTag (aka the Integrity Check Value) needs to be written
1123          * back to the packet. */
1124         if (one_entry_in_sg) {
1125                 if (unlikely(req->src != req->dst)) {
1126                         scatterwalk_unmap(dst, 0);
1127                         scatterwalk_done(&dst_sg_walk, 0, 0);
1128                 }
1129                 scatterwalk_unmap(src, 0);
1130                 scatterwalk_unmap(assoc, 0);
1131                 scatterwalk_done(&src_sg_walk, 0, 0);
1132                 scatterwalk_done(&assoc_sg_walk, 0, 0);
1133         } else {
1134                 scatterwalk_map_and_copy(dst, req->dst, 0,
1135                         req->cryptlen + auth_tag_len, 1);
1136                 kfree(src);
1137         }
1138         return 0;
1139 }
1140
1141 static int __driver_rfc4106_decrypt(struct aead_request *req)
1142 {
1143         u8 one_entry_in_sg = 0;
1144         u8 *src, *dst, *assoc;
1145         unsigned long tempCipherLen = 0;
1146         __be32 counter = cpu_to_be32(1);
1147         int retval = 0;
1148         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1149         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
1150         void *aes_ctx = &(ctx->aes_key_expanded);
1151         unsigned long auth_tag_len = crypto_aead_authsize(tfm);
1152         u8 iv_and_authTag[32+AESNI_ALIGN];
1153         u8 *iv = (u8 *) PTR_ALIGN((u8 *)iv_and_authTag, AESNI_ALIGN);
1154         u8 *authTag = iv + 16;
1155         struct scatter_walk src_sg_walk;
1156         struct scatter_walk assoc_sg_walk;
1157         struct scatter_walk dst_sg_walk;
1158         unsigned int i;
1159
1160         if (unlikely((req->cryptlen < auth_tag_len) ||
1161                 (req->assoclen != 8 && req->assoclen != 12)))
1162                 return -EINVAL;
1163         /* Assuming we are supporting rfc4106 64-bit extended */
1164         /* sequence numbers We need to have the AAD length */
1165         /* equal to 8 or 12 bytes */
1166
1167         tempCipherLen = (unsigned long)(req->cryptlen - auth_tag_len);
1168         /* IV below built */
1169         for (i = 0; i < 4; i++)
1170                 *(iv+i) = ctx->nonce[i];
1171         for (i = 0; i < 8; i++)
1172                 *(iv+4+i) = req->iv[i];
1173         *((__be32 *)(iv+12)) = counter;
1174
1175         if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) {
1176                 one_entry_in_sg = 1;
1177                 scatterwalk_start(&src_sg_walk, req->src);
1178                 scatterwalk_start(&assoc_sg_walk, req->assoc);
1179                 src = scatterwalk_map(&src_sg_walk, 0);
1180                 assoc = scatterwalk_map(&assoc_sg_walk, 0);
1181                 dst = src;
1182                 if (unlikely(req->src != req->dst)) {
1183                         scatterwalk_start(&dst_sg_walk, req->dst);
1184                         dst = scatterwalk_map(&dst_sg_walk, 0);
1185                 }
1186
1187         } else {
1188                 /* Allocate memory for src, dst, assoc */
1189                 src = kmalloc(req->cryptlen + req->assoclen, GFP_ATOMIC);
1190                 if (!src)
1191                         return -ENOMEM;
1192                 assoc = (src + req->cryptlen + auth_tag_len);
1193                 scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0);
1194                 scatterwalk_map_and_copy(assoc, req->assoc, 0,
1195                         req->assoclen, 0);
1196                 dst = src;
1197         }
1198
1199         aesni_gcm_dec(aes_ctx, dst, src, tempCipherLen, iv,
1200                 ctx->hash_subkey, assoc, (unsigned long)req->assoclen,
1201                 authTag, auth_tag_len);
1202
1203         /* Compare generated tag with passed in tag. */
1204         retval = memcmp(src + tempCipherLen, authTag, auth_tag_len) ?
1205                 -EBADMSG : 0;
1206
1207         if (one_entry_in_sg) {
1208                 if (unlikely(req->src != req->dst)) {
1209                         scatterwalk_unmap(dst, 0);
1210                         scatterwalk_done(&dst_sg_walk, 0, 0);
1211                 }
1212                 scatterwalk_unmap(src, 0);
1213                 scatterwalk_unmap(assoc, 0);
1214                 scatterwalk_done(&src_sg_walk, 0, 0);
1215                 scatterwalk_done(&assoc_sg_walk, 0, 0);
1216         } else {
1217                 scatterwalk_map_and_copy(dst, req->dst, 0, req->cryptlen, 1);
1218                 kfree(src);
1219         }
1220         return retval;
1221 }
1222
1223 static struct crypto_alg __rfc4106_alg = {
1224         .cra_name               = "__gcm-aes-aesni",
1225         .cra_driver_name        = "__driver-gcm-aes-aesni",
1226         .cra_priority           = 0,
1227         .cra_flags              = CRYPTO_ALG_TYPE_AEAD,
1228         .cra_blocksize          = 1,
1229         .cra_ctxsize    = sizeof(struct aesni_rfc4106_gcm_ctx) + AESNI_ALIGN,
1230         .cra_alignmask          = 0,
1231         .cra_type               = &crypto_aead_type,
1232         .cra_module             = THIS_MODULE,
1233         .cra_list               = LIST_HEAD_INIT(__rfc4106_alg.cra_list),
1234         .cra_u = {
1235                 .aead = {
1236                         .encrypt        = __driver_rfc4106_encrypt,
1237                         .decrypt        = __driver_rfc4106_decrypt,
1238                 },
1239         },
1240 };
1241 #endif
1242
1243 static int __init aesni_init(void)
1244 {
1245         int err;
1246
1247         if (!cpu_has_aes) {
1248                 printk(KERN_INFO "Intel AES-NI instructions are not detected.\n");
1249                 return -ENODEV;
1250         }
1251
1252         if ((err = crypto_register_alg(&aesni_alg)))
1253                 goto aes_err;
1254         if ((err = crypto_register_alg(&__aesni_alg)))
1255                 goto __aes_err;
1256         if ((err = crypto_register_alg(&blk_ecb_alg)))
1257                 goto blk_ecb_err;
1258         if ((err = crypto_register_alg(&blk_cbc_alg)))
1259                 goto blk_cbc_err;
1260         if ((err = crypto_register_alg(&ablk_ecb_alg)))
1261                 goto ablk_ecb_err;
1262         if ((err = crypto_register_alg(&ablk_cbc_alg)))
1263                 goto ablk_cbc_err;
1264 #ifdef CONFIG_X86_64
1265         if ((err = crypto_register_alg(&blk_ctr_alg)))
1266                 goto blk_ctr_err;
1267         if ((err = crypto_register_alg(&ablk_ctr_alg)))
1268                 goto ablk_ctr_err;
1269         if ((err = crypto_register_alg(&__rfc4106_alg)))
1270                 goto __aead_gcm_err;
1271         if ((err = crypto_register_alg(&rfc4106_alg)))
1272                 goto aead_gcm_err;
1273 #ifdef HAS_CTR
1274         if ((err = crypto_register_alg(&ablk_rfc3686_ctr_alg)))
1275                 goto ablk_rfc3686_ctr_err;
1276 #endif
1277 #endif
1278 #ifdef HAS_LRW
1279         if ((err = crypto_register_alg(&ablk_lrw_alg)))
1280                 goto ablk_lrw_err;
1281 #endif
1282 #ifdef HAS_PCBC
1283         if ((err = crypto_register_alg(&ablk_pcbc_alg)))
1284                 goto ablk_pcbc_err;
1285 #endif
1286 #ifdef HAS_XTS
1287         if ((err = crypto_register_alg(&ablk_xts_alg)))
1288                 goto ablk_xts_err;
1289 #endif
1290         return err;
1291
1292 #ifdef HAS_XTS
1293 ablk_xts_err:
1294 #endif
1295 #ifdef HAS_PCBC
1296         crypto_unregister_alg(&ablk_pcbc_alg);
1297 ablk_pcbc_err:
1298 #endif
1299 #ifdef HAS_LRW
1300         crypto_unregister_alg(&ablk_lrw_alg);
1301 ablk_lrw_err:
1302 #endif
1303 #ifdef CONFIG_X86_64
1304 #ifdef HAS_CTR
1305         crypto_unregister_alg(&ablk_rfc3686_ctr_alg);
1306 ablk_rfc3686_ctr_err:
1307 #endif
1308         crypto_unregister_alg(&rfc4106_alg);
1309 aead_gcm_err:
1310         crypto_unregister_alg(&__rfc4106_alg);
1311 __aead_gcm_err:
1312         crypto_unregister_alg(&ablk_ctr_alg);
1313 ablk_ctr_err:
1314         crypto_unregister_alg(&blk_ctr_alg);
1315 blk_ctr_err:
1316 #endif
1317         crypto_unregister_alg(&ablk_cbc_alg);
1318 ablk_cbc_err:
1319         crypto_unregister_alg(&ablk_ecb_alg);
1320 ablk_ecb_err:
1321         crypto_unregister_alg(&blk_cbc_alg);
1322 blk_cbc_err:
1323         crypto_unregister_alg(&blk_ecb_alg);
1324 blk_ecb_err:
1325         crypto_unregister_alg(&__aesni_alg);
1326 __aes_err:
1327         crypto_unregister_alg(&aesni_alg);
1328 aes_err:
1329         return err;
1330 }
1331
1332 static void __exit aesni_exit(void)
1333 {
1334 #ifdef HAS_XTS
1335         crypto_unregister_alg(&ablk_xts_alg);
1336 #endif
1337 #ifdef HAS_PCBC
1338         crypto_unregister_alg(&ablk_pcbc_alg);
1339 #endif
1340 #ifdef HAS_LRW
1341         crypto_unregister_alg(&ablk_lrw_alg);
1342 #endif
1343 #ifdef CONFIG_X86_64
1344 #ifdef HAS_CTR
1345         crypto_unregister_alg(&ablk_rfc3686_ctr_alg);
1346 #endif
1347         crypto_unregister_alg(&rfc4106_alg);
1348         crypto_unregister_alg(&__rfc4106_alg);
1349         crypto_unregister_alg(&ablk_ctr_alg);
1350         crypto_unregister_alg(&blk_ctr_alg);
1351 #endif
1352         crypto_unregister_alg(&ablk_cbc_alg);
1353         crypto_unregister_alg(&ablk_ecb_alg);
1354         crypto_unregister_alg(&blk_cbc_alg);
1355         crypto_unregister_alg(&blk_ecb_alg);
1356         crypto_unregister_alg(&__aesni_alg);
1357         crypto_unregister_alg(&aesni_alg);
1358 }
1359
1360 module_init(aesni_init);
1361 module_exit(aesni_exit);
1362
1363 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized");
1364 MODULE_LICENSE("GPL");
1365 MODULE_ALIAS("aes");