Merge branch 'samsung/exynos5' into next/soc2
[linux-2.6.git] / arch / x86 / crypto / aesni-intel_glue.c
1 /*
2  * Support for Intel AES-NI instructions. This file contains glue
3  * code, the real AES implementation is in intel-aes_asm.S.
4  *
5  * Copyright (C) 2008, Intel Corp.
6  *    Author: Huang Ying <ying.huang@intel.com>
7  *
8  * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
9  * interface for 64-bit kernels.
10  *    Authors: Adrian Hoban <adrian.hoban@intel.com>
11  *             Gabriele Paoloni <gabriele.paoloni@intel.com>
12  *             Tadeusz Struk (tadeusz.struk@intel.com)
13  *             Aidan O'Mahony (aidan.o.mahony@intel.com)
14  *    Copyright (c) 2010, Intel Corporation.
15  *
16  * This program is free software; you can redistribute it and/or modify
17  * it under the terms of the GNU General Public License as published by
18  * the Free Software Foundation; either version 2 of the License, or
19  * (at your option) any later version.
20  */
21
22 #include <linux/hardirq.h>
23 #include <linux/types.h>
24 #include <linux/crypto.h>
25 #include <linux/module.h>
26 #include <linux/err.h>
27 #include <crypto/algapi.h>
28 #include <crypto/aes.h>
29 #include <crypto/cryptd.h>
30 #include <crypto/ctr.h>
31 #include <asm/cpu_device_id.h>
32 #include <asm/i387.h>
33 #include <asm/aes.h>
34 #include <crypto/scatterwalk.h>
35 #include <crypto/internal/aead.h>
36 #include <linux/workqueue.h>
37 #include <linux/spinlock.h>
38
39 #if defined(CONFIG_CRYPTO_CTR) || defined(CONFIG_CRYPTO_CTR_MODULE)
40 #define HAS_CTR
41 #endif
42
43 #if defined(CONFIG_CRYPTO_LRW) || defined(CONFIG_CRYPTO_LRW_MODULE)
44 #define HAS_LRW
45 #endif
46
47 #if defined(CONFIG_CRYPTO_PCBC) || defined(CONFIG_CRYPTO_PCBC_MODULE)
48 #define HAS_PCBC
49 #endif
50
51 #if defined(CONFIG_CRYPTO_XTS) || defined(CONFIG_CRYPTO_XTS_MODULE)
52 #define HAS_XTS
53 #endif
54
55 struct async_aes_ctx {
56         struct cryptd_ablkcipher *cryptd_tfm;
57 };
58
59 /* This data is stored at the end of the crypto_tfm struct.
60  * It's a type of per "session" data storage location.
61  * This needs to be 16 byte aligned.
62  */
63 struct aesni_rfc4106_gcm_ctx {
64         u8 hash_subkey[16];
65         struct crypto_aes_ctx aes_key_expanded;
66         u8 nonce[4];
67         struct cryptd_aead *cryptd_tfm;
68 };
69
70 struct aesni_gcm_set_hash_subkey_result {
71         int err;
72         struct completion completion;
73 };
74
75 struct aesni_hash_subkey_req_data {
76         u8 iv[16];
77         struct aesni_gcm_set_hash_subkey_result result;
78         struct scatterlist sg;
79 };
80
81 #define AESNI_ALIGN     (16)
82 #define AES_BLOCK_MASK  (~(AES_BLOCK_SIZE-1))
83 #define RFC4106_HASH_SUBKEY_SIZE 16
84
85 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
86                              unsigned int key_len);
87 asmlinkage void aesni_enc(struct crypto_aes_ctx *ctx, u8 *out,
88                           const u8 *in);
89 asmlinkage void aesni_dec(struct crypto_aes_ctx *ctx, u8 *out,
90                           const u8 *in);
91 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out,
92                               const u8 *in, unsigned int len);
93 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out,
94                               const u8 *in, unsigned int len);
95 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
96                               const u8 *in, unsigned int len, u8 *iv);
97 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
98                               const u8 *in, unsigned int len, u8 *iv);
99
100 int crypto_fpu_init(void);
101 void crypto_fpu_exit(void);
102
103 #ifdef CONFIG_X86_64
104 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out,
105                               const u8 *in, unsigned int len, u8 *iv);
106
107 /* asmlinkage void aesni_gcm_enc()
108  * void *ctx,  AES Key schedule. Starts on a 16 byte boundary.
109  * u8 *out, Ciphertext output. Encrypt in-place is allowed.
110  * const u8 *in, Plaintext input
111  * unsigned long plaintext_len, Length of data in bytes for encryption.
112  * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association)
113  *         concatenated with 8 byte Initialisation Vector (from IPSec ESP
114  *         Payload) concatenated with 0x00000001. 16-byte aligned pointer.
115  * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
116  * const u8 *aad, Additional Authentication Data (AAD)
117  * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this
118  *          is going to be 8 or 12 bytes
119  * u8 *auth_tag, Authenticated Tag output.
120  * unsigned long auth_tag_len), Authenticated Tag Length in bytes.
121  *          Valid values are 16 (most likely), 12 or 8.
122  */
123 asmlinkage void aesni_gcm_enc(void *ctx, u8 *out,
124                         const u8 *in, unsigned long plaintext_len, u8 *iv,
125                         u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
126                         u8 *auth_tag, unsigned long auth_tag_len);
127
128 /* asmlinkage void aesni_gcm_dec()
129  * void *ctx, AES Key schedule. Starts on a 16 byte boundary.
130  * u8 *out, Plaintext output. Decrypt in-place is allowed.
131  * const u8 *in, Ciphertext input
132  * unsigned long ciphertext_len, Length of data in bytes for decryption.
133  * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association)
134  *         concatenated with 8 byte Initialisation Vector (from IPSec ESP
135  *         Payload) concatenated with 0x00000001. 16-byte aligned pointer.
136  * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
137  * const u8 *aad, Additional Authentication Data (AAD)
138  * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this is going
139  * to be 8 or 12 bytes
140  * u8 *auth_tag, Authenticated Tag output.
141  * unsigned long auth_tag_len) Authenticated Tag Length in bytes.
142  * Valid values are 16 (most likely), 12 or 8.
143  */
144 asmlinkage void aesni_gcm_dec(void *ctx, u8 *out,
145                         const u8 *in, unsigned long ciphertext_len, u8 *iv,
146                         u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
147                         u8 *auth_tag, unsigned long auth_tag_len);
148
149 static inline struct
150 aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm)
151 {
152         return
153                 (struct aesni_rfc4106_gcm_ctx *)
154                 PTR_ALIGN((u8 *)
155                 crypto_tfm_ctx(crypto_aead_tfm(tfm)), AESNI_ALIGN);
156 }
157 #endif
158
159 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
160 {
161         unsigned long addr = (unsigned long)raw_ctx;
162         unsigned long align = AESNI_ALIGN;
163
164         if (align <= crypto_tfm_ctx_alignment())
165                 align = 1;
166         return (struct crypto_aes_ctx *)ALIGN(addr, align);
167 }
168
169 static int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx,
170                               const u8 *in_key, unsigned int key_len)
171 {
172         struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx);
173         u32 *flags = &tfm->crt_flags;
174         int err;
175
176         if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 &&
177             key_len != AES_KEYSIZE_256) {
178                 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
179                 return -EINVAL;
180         }
181
182         if (!irq_fpu_usable())
183                 err = crypto_aes_expand_key(ctx, in_key, key_len);
184         else {
185                 kernel_fpu_begin();
186                 err = aesni_set_key(ctx, in_key, key_len);
187                 kernel_fpu_end();
188         }
189
190         return err;
191 }
192
193 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
194                        unsigned int key_len)
195 {
196         return aes_set_key_common(tfm, crypto_tfm_ctx(tfm), in_key, key_len);
197 }
198
199 static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
200 {
201         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
202
203         if (!irq_fpu_usable())
204                 crypto_aes_encrypt_x86(ctx, dst, src);
205         else {
206                 kernel_fpu_begin();
207                 aesni_enc(ctx, dst, src);
208                 kernel_fpu_end();
209         }
210 }
211
212 static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
213 {
214         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
215
216         if (!irq_fpu_usable())
217                 crypto_aes_decrypt_x86(ctx, dst, src);
218         else {
219                 kernel_fpu_begin();
220                 aesni_dec(ctx, dst, src);
221                 kernel_fpu_end();
222         }
223 }
224
225 static struct crypto_alg aesni_alg = {
226         .cra_name               = "aes",
227         .cra_driver_name        = "aes-aesni",
228         .cra_priority           = 300,
229         .cra_flags              = CRYPTO_ALG_TYPE_CIPHER,
230         .cra_blocksize          = AES_BLOCK_SIZE,
231         .cra_ctxsize            = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
232         .cra_alignmask          = 0,
233         .cra_module             = THIS_MODULE,
234         .cra_list               = LIST_HEAD_INIT(aesni_alg.cra_list),
235         .cra_u  = {
236                 .cipher = {
237                         .cia_min_keysize        = AES_MIN_KEY_SIZE,
238                         .cia_max_keysize        = AES_MAX_KEY_SIZE,
239                         .cia_setkey             = aes_set_key,
240                         .cia_encrypt            = aes_encrypt,
241                         .cia_decrypt            = aes_decrypt
242                 }
243         }
244 };
245
246 static void __aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
247 {
248         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
249
250         aesni_enc(ctx, dst, src);
251 }
252
253 static void __aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
254 {
255         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
256
257         aesni_dec(ctx, dst, src);
258 }
259
260 static struct crypto_alg __aesni_alg = {
261         .cra_name               = "__aes-aesni",
262         .cra_driver_name        = "__driver-aes-aesni",
263         .cra_priority           = 0,
264         .cra_flags              = CRYPTO_ALG_TYPE_CIPHER,
265         .cra_blocksize          = AES_BLOCK_SIZE,
266         .cra_ctxsize            = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
267         .cra_alignmask          = 0,
268         .cra_module             = THIS_MODULE,
269         .cra_list               = LIST_HEAD_INIT(__aesni_alg.cra_list),
270         .cra_u  = {
271                 .cipher = {
272                         .cia_min_keysize        = AES_MIN_KEY_SIZE,
273                         .cia_max_keysize        = AES_MAX_KEY_SIZE,
274                         .cia_setkey             = aes_set_key,
275                         .cia_encrypt            = __aes_encrypt,
276                         .cia_decrypt            = __aes_decrypt
277                 }
278         }
279 };
280
281 static int ecb_encrypt(struct blkcipher_desc *desc,
282                        struct scatterlist *dst, struct scatterlist *src,
283                        unsigned int nbytes)
284 {
285         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
286         struct blkcipher_walk walk;
287         int err;
288
289         blkcipher_walk_init(&walk, dst, src, nbytes);
290         err = blkcipher_walk_virt(desc, &walk);
291         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
292
293         kernel_fpu_begin();
294         while ((nbytes = walk.nbytes)) {
295                 aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
296                               nbytes & AES_BLOCK_MASK);
297                 nbytes &= AES_BLOCK_SIZE - 1;
298                 err = blkcipher_walk_done(desc, &walk, nbytes);
299         }
300         kernel_fpu_end();
301
302         return err;
303 }
304
305 static int ecb_decrypt(struct blkcipher_desc *desc,
306                        struct scatterlist *dst, struct scatterlist *src,
307                        unsigned int nbytes)
308 {
309         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
310         struct blkcipher_walk walk;
311         int err;
312
313         blkcipher_walk_init(&walk, dst, src, nbytes);
314         err = blkcipher_walk_virt(desc, &walk);
315         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
316
317         kernel_fpu_begin();
318         while ((nbytes = walk.nbytes)) {
319                 aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
320                               nbytes & AES_BLOCK_MASK);
321                 nbytes &= AES_BLOCK_SIZE - 1;
322                 err = blkcipher_walk_done(desc, &walk, nbytes);
323         }
324         kernel_fpu_end();
325
326         return err;
327 }
328
329 static struct crypto_alg blk_ecb_alg = {
330         .cra_name               = "__ecb-aes-aesni",
331         .cra_driver_name        = "__driver-ecb-aes-aesni",
332         .cra_priority           = 0,
333         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
334         .cra_blocksize          = AES_BLOCK_SIZE,
335         .cra_ctxsize            = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
336         .cra_alignmask          = 0,
337         .cra_type               = &crypto_blkcipher_type,
338         .cra_module             = THIS_MODULE,
339         .cra_list               = LIST_HEAD_INIT(blk_ecb_alg.cra_list),
340         .cra_u = {
341                 .blkcipher = {
342                         .min_keysize    = AES_MIN_KEY_SIZE,
343                         .max_keysize    = AES_MAX_KEY_SIZE,
344                         .setkey         = aes_set_key,
345                         .encrypt        = ecb_encrypt,
346                         .decrypt        = ecb_decrypt,
347                 },
348         },
349 };
350
351 static int cbc_encrypt(struct blkcipher_desc *desc,
352                        struct scatterlist *dst, struct scatterlist *src,
353                        unsigned int nbytes)
354 {
355         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
356         struct blkcipher_walk walk;
357         int err;
358
359         blkcipher_walk_init(&walk, dst, src, nbytes);
360         err = blkcipher_walk_virt(desc, &walk);
361         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
362
363         kernel_fpu_begin();
364         while ((nbytes = walk.nbytes)) {
365                 aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
366                               nbytes & AES_BLOCK_MASK, walk.iv);
367                 nbytes &= AES_BLOCK_SIZE - 1;
368                 err = blkcipher_walk_done(desc, &walk, nbytes);
369         }
370         kernel_fpu_end();
371
372         return err;
373 }
374
375 static int cbc_decrypt(struct blkcipher_desc *desc,
376                        struct scatterlist *dst, struct scatterlist *src,
377                        unsigned int nbytes)
378 {
379         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
380         struct blkcipher_walk walk;
381         int err;
382
383         blkcipher_walk_init(&walk, dst, src, nbytes);
384         err = blkcipher_walk_virt(desc, &walk);
385         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
386
387         kernel_fpu_begin();
388         while ((nbytes = walk.nbytes)) {
389                 aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
390                               nbytes & AES_BLOCK_MASK, walk.iv);
391                 nbytes &= AES_BLOCK_SIZE - 1;
392                 err = blkcipher_walk_done(desc, &walk, nbytes);
393         }
394         kernel_fpu_end();
395
396         return err;
397 }
398
399 static struct crypto_alg blk_cbc_alg = {
400         .cra_name               = "__cbc-aes-aesni",
401         .cra_driver_name        = "__driver-cbc-aes-aesni",
402         .cra_priority           = 0,
403         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
404         .cra_blocksize          = AES_BLOCK_SIZE,
405         .cra_ctxsize            = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
406         .cra_alignmask          = 0,
407         .cra_type               = &crypto_blkcipher_type,
408         .cra_module             = THIS_MODULE,
409         .cra_list               = LIST_HEAD_INIT(blk_cbc_alg.cra_list),
410         .cra_u = {
411                 .blkcipher = {
412                         .min_keysize    = AES_MIN_KEY_SIZE,
413                         .max_keysize    = AES_MAX_KEY_SIZE,
414                         .setkey         = aes_set_key,
415                         .encrypt        = cbc_encrypt,
416                         .decrypt        = cbc_decrypt,
417                 },
418         },
419 };
420
421 #ifdef CONFIG_X86_64
422 static void ctr_crypt_final(struct crypto_aes_ctx *ctx,
423                             struct blkcipher_walk *walk)
424 {
425         u8 *ctrblk = walk->iv;
426         u8 keystream[AES_BLOCK_SIZE];
427         u8 *src = walk->src.virt.addr;
428         u8 *dst = walk->dst.virt.addr;
429         unsigned int nbytes = walk->nbytes;
430
431         aesni_enc(ctx, keystream, ctrblk);
432         crypto_xor(keystream, src, nbytes);
433         memcpy(dst, keystream, nbytes);
434         crypto_inc(ctrblk, AES_BLOCK_SIZE);
435 }
436
437 static int ctr_crypt(struct blkcipher_desc *desc,
438                      struct scatterlist *dst, struct scatterlist *src,
439                      unsigned int nbytes)
440 {
441         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
442         struct blkcipher_walk walk;
443         int err;
444
445         blkcipher_walk_init(&walk, dst, src, nbytes);
446         err = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE);
447         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
448
449         kernel_fpu_begin();
450         while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
451                 aesni_ctr_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
452                               nbytes & AES_BLOCK_MASK, walk.iv);
453                 nbytes &= AES_BLOCK_SIZE - 1;
454                 err = blkcipher_walk_done(desc, &walk, nbytes);
455         }
456         if (walk.nbytes) {
457                 ctr_crypt_final(ctx, &walk);
458                 err = blkcipher_walk_done(desc, &walk, 0);
459         }
460         kernel_fpu_end();
461
462         return err;
463 }
464
465 static struct crypto_alg blk_ctr_alg = {
466         .cra_name               = "__ctr-aes-aesni",
467         .cra_driver_name        = "__driver-ctr-aes-aesni",
468         .cra_priority           = 0,
469         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
470         .cra_blocksize          = 1,
471         .cra_ctxsize            = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
472         .cra_alignmask          = 0,
473         .cra_type               = &crypto_blkcipher_type,
474         .cra_module             = THIS_MODULE,
475         .cra_list               = LIST_HEAD_INIT(blk_ctr_alg.cra_list),
476         .cra_u = {
477                 .blkcipher = {
478                         .min_keysize    = AES_MIN_KEY_SIZE,
479                         .max_keysize    = AES_MAX_KEY_SIZE,
480                         .ivsize         = AES_BLOCK_SIZE,
481                         .setkey         = aes_set_key,
482                         .encrypt        = ctr_crypt,
483                         .decrypt        = ctr_crypt,
484                 },
485         },
486 };
487 #endif
488
489 static int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key,
490                         unsigned int key_len)
491 {
492         struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
493         struct crypto_ablkcipher *child = &ctx->cryptd_tfm->base;
494         int err;
495
496         crypto_ablkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
497         crypto_ablkcipher_set_flags(child, crypto_ablkcipher_get_flags(tfm)
498                                     & CRYPTO_TFM_REQ_MASK);
499         err = crypto_ablkcipher_setkey(child, key, key_len);
500         crypto_ablkcipher_set_flags(tfm, crypto_ablkcipher_get_flags(child)
501                                     & CRYPTO_TFM_RES_MASK);
502         return err;
503 }
504
505 static int ablk_encrypt(struct ablkcipher_request *req)
506 {
507         struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
508         struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
509
510         if (!irq_fpu_usable()) {
511                 struct ablkcipher_request *cryptd_req =
512                         ablkcipher_request_ctx(req);
513                 memcpy(cryptd_req, req, sizeof(*req));
514                 ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
515                 return crypto_ablkcipher_encrypt(cryptd_req);
516         } else {
517                 struct blkcipher_desc desc;
518                 desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm);
519                 desc.info = req->info;
520                 desc.flags = 0;
521                 return crypto_blkcipher_crt(desc.tfm)->encrypt(
522                         &desc, req->dst, req->src, req->nbytes);
523         }
524 }
525
526 static int ablk_decrypt(struct ablkcipher_request *req)
527 {
528         struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
529         struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
530
531         if (!irq_fpu_usable()) {
532                 struct ablkcipher_request *cryptd_req =
533                         ablkcipher_request_ctx(req);
534                 memcpy(cryptd_req, req, sizeof(*req));
535                 ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
536                 return crypto_ablkcipher_decrypt(cryptd_req);
537         } else {
538                 struct blkcipher_desc desc;
539                 desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm);
540                 desc.info = req->info;
541                 desc.flags = 0;
542                 return crypto_blkcipher_crt(desc.tfm)->decrypt(
543                         &desc, req->dst, req->src, req->nbytes);
544         }
545 }
546
547 static void ablk_exit(struct crypto_tfm *tfm)
548 {
549         struct async_aes_ctx *ctx = crypto_tfm_ctx(tfm);
550
551         cryptd_free_ablkcipher(ctx->cryptd_tfm);
552 }
553
554 static void ablk_init_common(struct crypto_tfm *tfm,
555                              struct cryptd_ablkcipher *cryptd_tfm)
556 {
557         struct async_aes_ctx *ctx = crypto_tfm_ctx(tfm);
558
559         ctx->cryptd_tfm = cryptd_tfm;
560         tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request) +
561                 crypto_ablkcipher_reqsize(&cryptd_tfm->base);
562 }
563
564 static int ablk_ecb_init(struct crypto_tfm *tfm)
565 {
566         struct cryptd_ablkcipher *cryptd_tfm;
567
568         cryptd_tfm = cryptd_alloc_ablkcipher("__driver-ecb-aes-aesni", 0, 0);
569         if (IS_ERR(cryptd_tfm))
570                 return PTR_ERR(cryptd_tfm);
571         ablk_init_common(tfm, cryptd_tfm);
572         return 0;
573 }
574
575 static struct crypto_alg ablk_ecb_alg = {
576         .cra_name               = "ecb(aes)",
577         .cra_driver_name        = "ecb-aes-aesni",
578         .cra_priority           = 400,
579         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
580         .cra_blocksize          = AES_BLOCK_SIZE,
581         .cra_ctxsize            = sizeof(struct async_aes_ctx),
582         .cra_alignmask          = 0,
583         .cra_type               = &crypto_ablkcipher_type,
584         .cra_module             = THIS_MODULE,
585         .cra_list               = LIST_HEAD_INIT(ablk_ecb_alg.cra_list),
586         .cra_init               = ablk_ecb_init,
587         .cra_exit               = ablk_exit,
588         .cra_u = {
589                 .ablkcipher = {
590                         .min_keysize    = AES_MIN_KEY_SIZE,
591                         .max_keysize    = AES_MAX_KEY_SIZE,
592                         .setkey         = ablk_set_key,
593                         .encrypt        = ablk_encrypt,
594                         .decrypt        = ablk_decrypt,
595                 },
596         },
597 };
598
599 static int ablk_cbc_init(struct crypto_tfm *tfm)
600 {
601         struct cryptd_ablkcipher *cryptd_tfm;
602
603         cryptd_tfm = cryptd_alloc_ablkcipher("__driver-cbc-aes-aesni", 0, 0);
604         if (IS_ERR(cryptd_tfm))
605                 return PTR_ERR(cryptd_tfm);
606         ablk_init_common(tfm, cryptd_tfm);
607         return 0;
608 }
609
610 static struct crypto_alg ablk_cbc_alg = {
611         .cra_name               = "cbc(aes)",
612         .cra_driver_name        = "cbc-aes-aesni",
613         .cra_priority           = 400,
614         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
615         .cra_blocksize          = AES_BLOCK_SIZE,
616         .cra_ctxsize            = sizeof(struct async_aes_ctx),
617         .cra_alignmask          = 0,
618         .cra_type               = &crypto_ablkcipher_type,
619         .cra_module             = THIS_MODULE,
620         .cra_list               = LIST_HEAD_INIT(ablk_cbc_alg.cra_list),
621         .cra_init               = ablk_cbc_init,
622         .cra_exit               = ablk_exit,
623         .cra_u = {
624                 .ablkcipher = {
625                         .min_keysize    = AES_MIN_KEY_SIZE,
626                         .max_keysize    = AES_MAX_KEY_SIZE,
627                         .ivsize         = AES_BLOCK_SIZE,
628                         .setkey         = ablk_set_key,
629                         .encrypt        = ablk_encrypt,
630                         .decrypt        = ablk_decrypt,
631                 },
632         },
633 };
634
635 #ifdef CONFIG_X86_64
636 static int ablk_ctr_init(struct crypto_tfm *tfm)
637 {
638         struct cryptd_ablkcipher *cryptd_tfm;
639
640         cryptd_tfm = cryptd_alloc_ablkcipher("__driver-ctr-aes-aesni", 0, 0);
641         if (IS_ERR(cryptd_tfm))
642                 return PTR_ERR(cryptd_tfm);
643         ablk_init_common(tfm, cryptd_tfm);
644         return 0;
645 }
646
647 static struct crypto_alg ablk_ctr_alg = {
648         .cra_name               = "ctr(aes)",
649         .cra_driver_name        = "ctr-aes-aesni",
650         .cra_priority           = 400,
651         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
652         .cra_blocksize          = 1,
653         .cra_ctxsize            = sizeof(struct async_aes_ctx),
654         .cra_alignmask          = 0,
655         .cra_type               = &crypto_ablkcipher_type,
656         .cra_module             = THIS_MODULE,
657         .cra_list               = LIST_HEAD_INIT(ablk_ctr_alg.cra_list),
658         .cra_init               = ablk_ctr_init,
659         .cra_exit               = ablk_exit,
660         .cra_u = {
661                 .ablkcipher = {
662                         .min_keysize    = AES_MIN_KEY_SIZE,
663                         .max_keysize    = AES_MAX_KEY_SIZE,
664                         .ivsize         = AES_BLOCK_SIZE,
665                         .setkey         = ablk_set_key,
666                         .encrypt        = ablk_encrypt,
667                         .decrypt        = ablk_encrypt,
668                         .geniv          = "chainiv",
669                 },
670         },
671 };
672
673 #ifdef HAS_CTR
674 static int ablk_rfc3686_ctr_init(struct crypto_tfm *tfm)
675 {
676         struct cryptd_ablkcipher *cryptd_tfm;
677
678         cryptd_tfm = cryptd_alloc_ablkcipher(
679                 "rfc3686(__driver-ctr-aes-aesni)", 0, 0);
680         if (IS_ERR(cryptd_tfm))
681                 return PTR_ERR(cryptd_tfm);
682         ablk_init_common(tfm, cryptd_tfm);
683         return 0;
684 }
685
686 static struct crypto_alg ablk_rfc3686_ctr_alg = {
687         .cra_name               = "rfc3686(ctr(aes))",
688         .cra_driver_name        = "rfc3686-ctr-aes-aesni",
689         .cra_priority           = 400,
690         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
691         .cra_blocksize          = 1,
692         .cra_ctxsize            = sizeof(struct async_aes_ctx),
693         .cra_alignmask          = 0,
694         .cra_type               = &crypto_ablkcipher_type,
695         .cra_module             = THIS_MODULE,
696         .cra_list               = LIST_HEAD_INIT(ablk_rfc3686_ctr_alg.cra_list),
697         .cra_init               = ablk_rfc3686_ctr_init,
698         .cra_exit               = ablk_exit,
699         .cra_u = {
700                 .ablkcipher = {
701                         .min_keysize = AES_MIN_KEY_SIZE+CTR_RFC3686_NONCE_SIZE,
702                         .max_keysize = AES_MAX_KEY_SIZE+CTR_RFC3686_NONCE_SIZE,
703                         .ivsize      = CTR_RFC3686_IV_SIZE,
704                         .setkey      = ablk_set_key,
705                         .encrypt     = ablk_encrypt,
706                         .decrypt     = ablk_decrypt,
707                         .geniv       = "seqiv",
708                 },
709         },
710 };
711 #endif
712 #endif
713
714 #ifdef HAS_LRW
715 static int ablk_lrw_init(struct crypto_tfm *tfm)
716 {
717         struct cryptd_ablkcipher *cryptd_tfm;
718
719         cryptd_tfm = cryptd_alloc_ablkcipher("fpu(lrw(__driver-aes-aesni))",
720                                              0, 0);
721         if (IS_ERR(cryptd_tfm))
722                 return PTR_ERR(cryptd_tfm);
723         ablk_init_common(tfm, cryptd_tfm);
724         return 0;
725 }
726
727 static struct crypto_alg ablk_lrw_alg = {
728         .cra_name               = "lrw(aes)",
729         .cra_driver_name        = "lrw-aes-aesni",
730         .cra_priority           = 400,
731         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
732         .cra_blocksize          = AES_BLOCK_SIZE,
733         .cra_ctxsize            = sizeof(struct async_aes_ctx),
734         .cra_alignmask          = 0,
735         .cra_type               = &crypto_ablkcipher_type,
736         .cra_module             = THIS_MODULE,
737         .cra_list               = LIST_HEAD_INIT(ablk_lrw_alg.cra_list),
738         .cra_init               = ablk_lrw_init,
739         .cra_exit               = ablk_exit,
740         .cra_u = {
741                 .ablkcipher = {
742                         .min_keysize    = AES_MIN_KEY_SIZE + AES_BLOCK_SIZE,
743                         .max_keysize    = AES_MAX_KEY_SIZE + AES_BLOCK_SIZE,
744                         .ivsize         = AES_BLOCK_SIZE,
745                         .setkey         = ablk_set_key,
746                         .encrypt        = ablk_encrypt,
747                         .decrypt        = ablk_decrypt,
748                 },
749         },
750 };
751 #endif
752
753 #ifdef HAS_PCBC
754 static int ablk_pcbc_init(struct crypto_tfm *tfm)
755 {
756         struct cryptd_ablkcipher *cryptd_tfm;
757
758         cryptd_tfm = cryptd_alloc_ablkcipher("fpu(pcbc(__driver-aes-aesni))",
759                                              0, 0);
760         if (IS_ERR(cryptd_tfm))
761                 return PTR_ERR(cryptd_tfm);
762         ablk_init_common(tfm, cryptd_tfm);
763         return 0;
764 }
765
766 static struct crypto_alg ablk_pcbc_alg = {
767         .cra_name               = "pcbc(aes)",
768         .cra_driver_name        = "pcbc-aes-aesni",
769         .cra_priority           = 400,
770         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
771         .cra_blocksize          = AES_BLOCK_SIZE,
772         .cra_ctxsize            = sizeof(struct async_aes_ctx),
773         .cra_alignmask          = 0,
774         .cra_type               = &crypto_ablkcipher_type,
775         .cra_module             = THIS_MODULE,
776         .cra_list               = LIST_HEAD_INIT(ablk_pcbc_alg.cra_list),
777         .cra_init               = ablk_pcbc_init,
778         .cra_exit               = ablk_exit,
779         .cra_u = {
780                 .ablkcipher = {
781                         .min_keysize    = AES_MIN_KEY_SIZE,
782                         .max_keysize    = AES_MAX_KEY_SIZE,
783                         .ivsize         = AES_BLOCK_SIZE,
784                         .setkey         = ablk_set_key,
785                         .encrypt        = ablk_encrypt,
786                         .decrypt        = ablk_decrypt,
787                 },
788         },
789 };
790 #endif
791
792 #ifdef HAS_XTS
793 static int ablk_xts_init(struct crypto_tfm *tfm)
794 {
795         struct cryptd_ablkcipher *cryptd_tfm;
796
797         cryptd_tfm = cryptd_alloc_ablkcipher("fpu(xts(__driver-aes-aesni))",
798                                              0, 0);
799         if (IS_ERR(cryptd_tfm))
800                 return PTR_ERR(cryptd_tfm);
801         ablk_init_common(tfm, cryptd_tfm);
802         return 0;
803 }
804
805 static struct crypto_alg ablk_xts_alg = {
806         .cra_name               = "xts(aes)",
807         .cra_driver_name        = "xts-aes-aesni",
808         .cra_priority           = 400,
809         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
810         .cra_blocksize          = AES_BLOCK_SIZE,
811         .cra_ctxsize            = sizeof(struct async_aes_ctx),
812         .cra_alignmask          = 0,
813         .cra_type               = &crypto_ablkcipher_type,
814         .cra_module             = THIS_MODULE,
815         .cra_list               = LIST_HEAD_INIT(ablk_xts_alg.cra_list),
816         .cra_init               = ablk_xts_init,
817         .cra_exit               = ablk_exit,
818         .cra_u = {
819                 .ablkcipher = {
820                         .min_keysize    = 2 * AES_MIN_KEY_SIZE,
821                         .max_keysize    = 2 * AES_MAX_KEY_SIZE,
822                         .ivsize         = AES_BLOCK_SIZE,
823                         .setkey         = ablk_set_key,
824                         .encrypt        = ablk_encrypt,
825                         .decrypt        = ablk_decrypt,
826                 },
827         },
828 };
829 #endif
830
831 #ifdef CONFIG_X86_64
832 static int rfc4106_init(struct crypto_tfm *tfm)
833 {
834         struct cryptd_aead *cryptd_tfm;
835         struct aesni_rfc4106_gcm_ctx *ctx = (struct aesni_rfc4106_gcm_ctx *)
836                 PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN);
837         struct crypto_aead *cryptd_child;
838         struct aesni_rfc4106_gcm_ctx *child_ctx;
839         cryptd_tfm = cryptd_alloc_aead("__driver-gcm-aes-aesni", 0, 0);
840         if (IS_ERR(cryptd_tfm))
841                 return PTR_ERR(cryptd_tfm);
842
843         cryptd_child = cryptd_aead_child(cryptd_tfm);
844         child_ctx = aesni_rfc4106_gcm_ctx_get(cryptd_child);
845         memcpy(child_ctx, ctx, sizeof(*ctx));
846         ctx->cryptd_tfm = cryptd_tfm;
847         tfm->crt_aead.reqsize = sizeof(struct aead_request)
848                 + crypto_aead_reqsize(&cryptd_tfm->base);
849         return 0;
850 }
851
852 static void rfc4106_exit(struct crypto_tfm *tfm)
853 {
854         struct aesni_rfc4106_gcm_ctx *ctx =
855                 (struct aesni_rfc4106_gcm_ctx *)
856                 PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN);
857         if (!IS_ERR(ctx->cryptd_tfm))
858                 cryptd_free_aead(ctx->cryptd_tfm);
859         return;
860 }
861
862 static void
863 rfc4106_set_hash_subkey_done(struct crypto_async_request *req, int err)
864 {
865         struct aesni_gcm_set_hash_subkey_result *result = req->data;
866
867         if (err == -EINPROGRESS)
868                 return;
869         result->err = err;
870         complete(&result->completion);
871 }
872
873 static int
874 rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len)
875 {
876         struct crypto_ablkcipher *ctr_tfm;
877         struct ablkcipher_request *req;
878         int ret = -EINVAL;
879         struct aesni_hash_subkey_req_data *req_data;
880
881         ctr_tfm = crypto_alloc_ablkcipher("ctr(aes)", 0, 0);
882         if (IS_ERR(ctr_tfm))
883                 return PTR_ERR(ctr_tfm);
884
885         crypto_ablkcipher_clear_flags(ctr_tfm, ~0);
886
887         ret = crypto_ablkcipher_setkey(ctr_tfm, key, key_len);
888         if (ret)
889                 goto out_free_ablkcipher;
890
891         ret = -ENOMEM;
892         req = ablkcipher_request_alloc(ctr_tfm, GFP_KERNEL);
893         if (!req)
894                 goto out_free_ablkcipher;
895
896         req_data = kmalloc(sizeof(*req_data), GFP_KERNEL);
897         if (!req_data)
898                 goto out_free_request;
899
900         memset(req_data->iv, 0, sizeof(req_data->iv));
901
902         /* Clear the data in the hash sub key container to zero.*/
903         /* We want to cipher all zeros to create the hash sub key. */
904         memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE);
905
906         init_completion(&req_data->result.completion);
907         sg_init_one(&req_data->sg, hash_subkey, RFC4106_HASH_SUBKEY_SIZE);
908         ablkcipher_request_set_tfm(req, ctr_tfm);
909         ablkcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP |
910                                         CRYPTO_TFM_REQ_MAY_BACKLOG,
911                                         rfc4106_set_hash_subkey_done,
912                                         &req_data->result);
913
914         ablkcipher_request_set_crypt(req, &req_data->sg,
915                 &req_data->sg, RFC4106_HASH_SUBKEY_SIZE, req_data->iv);
916
917         ret = crypto_ablkcipher_encrypt(req);
918         if (ret == -EINPROGRESS || ret == -EBUSY) {
919                 ret = wait_for_completion_interruptible
920                         (&req_data->result.completion);
921                 if (!ret)
922                         ret = req_data->result.err;
923         }
924         kfree(req_data);
925 out_free_request:
926         ablkcipher_request_free(req);
927 out_free_ablkcipher:
928         crypto_free_ablkcipher(ctr_tfm);
929         return ret;
930 }
931
932 static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key,
933                                                    unsigned int key_len)
934 {
935         int ret = 0;
936         struct crypto_tfm *tfm = crypto_aead_tfm(parent);
937         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent);
938         struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
939         struct aesni_rfc4106_gcm_ctx *child_ctx =
940                                  aesni_rfc4106_gcm_ctx_get(cryptd_child);
941         u8 *new_key_mem = NULL;
942
943         if (key_len < 4) {
944                 crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
945                 return -EINVAL;
946         }
947         /*Account for 4 byte nonce at the end.*/
948         key_len -= 4;
949         if (key_len != AES_KEYSIZE_128) {
950                 crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
951                 return -EINVAL;
952         }
953
954         memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce));
955         /*This must be on a 16 byte boundary!*/
956         if ((unsigned long)(&(ctx->aes_key_expanded.key_enc[0])) % AESNI_ALIGN)
957                 return -EINVAL;
958
959         if ((unsigned long)key % AESNI_ALIGN) {
960                 /*key is not aligned: use an auxuliar aligned pointer*/
961                 new_key_mem = kmalloc(key_len+AESNI_ALIGN, GFP_KERNEL);
962                 if (!new_key_mem)
963                         return -ENOMEM;
964
965                 new_key_mem = PTR_ALIGN(new_key_mem, AESNI_ALIGN);
966                 memcpy(new_key_mem, key, key_len);
967                 key = new_key_mem;
968         }
969
970         if (!irq_fpu_usable())
971                 ret = crypto_aes_expand_key(&(ctx->aes_key_expanded),
972                 key, key_len);
973         else {
974                 kernel_fpu_begin();
975                 ret = aesni_set_key(&(ctx->aes_key_expanded), key, key_len);
976                 kernel_fpu_end();
977         }
978         /*This must be on a 16 byte boundary!*/
979         if ((unsigned long)(&(ctx->hash_subkey[0])) % AESNI_ALIGN) {
980                 ret = -EINVAL;
981                 goto exit;
982         }
983         ret = rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
984         memcpy(child_ctx, ctx, sizeof(*ctx));
985 exit:
986         kfree(new_key_mem);
987         return ret;
988 }
989
990 /* This is the Integrity Check Value (aka the authentication tag length and can
991  * be 8, 12 or 16 bytes long. */
992 static int rfc4106_set_authsize(struct crypto_aead *parent,
993                                 unsigned int authsize)
994 {
995         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent);
996         struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
997
998         switch (authsize) {
999         case 8:
1000         case 12:
1001         case 16:
1002                 break;
1003         default:
1004                 return -EINVAL;
1005         }
1006         crypto_aead_crt(parent)->authsize = authsize;
1007         crypto_aead_crt(cryptd_child)->authsize = authsize;
1008         return 0;
1009 }
1010
1011 static int rfc4106_encrypt(struct aead_request *req)
1012 {
1013         int ret;
1014         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1015         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
1016
1017         if (!irq_fpu_usable()) {
1018                 struct aead_request *cryptd_req =
1019                         (struct aead_request *) aead_request_ctx(req);
1020                 memcpy(cryptd_req, req, sizeof(*req));
1021                 aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
1022                 return crypto_aead_encrypt(cryptd_req);
1023         } else {
1024                 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
1025                 kernel_fpu_begin();
1026                 ret = cryptd_child->base.crt_aead.encrypt(req);
1027                 kernel_fpu_end();
1028                 return ret;
1029         }
1030 }
1031
1032 static int rfc4106_decrypt(struct aead_request *req)
1033 {
1034         int ret;
1035         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1036         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
1037
1038         if (!irq_fpu_usable()) {
1039                 struct aead_request *cryptd_req =
1040                         (struct aead_request *) aead_request_ctx(req);
1041                 memcpy(cryptd_req, req, sizeof(*req));
1042                 aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
1043                 return crypto_aead_decrypt(cryptd_req);
1044         } else {
1045                 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
1046                 kernel_fpu_begin();
1047                 ret = cryptd_child->base.crt_aead.decrypt(req);
1048                 kernel_fpu_end();
1049                 return ret;
1050         }
1051 }
1052
1053 static struct crypto_alg rfc4106_alg = {
1054         .cra_name = "rfc4106(gcm(aes))",
1055         .cra_driver_name = "rfc4106-gcm-aesni",
1056         .cra_priority = 400,
1057         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
1058         .cra_blocksize = 1,
1059         .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx) + AESNI_ALIGN,
1060         .cra_alignmask = 0,
1061         .cra_type = &crypto_nivaead_type,
1062         .cra_module = THIS_MODULE,
1063         .cra_list = LIST_HEAD_INIT(rfc4106_alg.cra_list),
1064         .cra_init = rfc4106_init,
1065         .cra_exit = rfc4106_exit,
1066         .cra_u = {
1067                 .aead = {
1068                         .setkey = rfc4106_set_key,
1069                         .setauthsize = rfc4106_set_authsize,
1070                         .encrypt = rfc4106_encrypt,
1071                         .decrypt = rfc4106_decrypt,
1072                         .geniv = "seqiv",
1073                         .ivsize = 8,
1074                         .maxauthsize = 16,
1075                 },
1076         },
1077 };
1078
1079 static int __driver_rfc4106_encrypt(struct aead_request *req)
1080 {
1081         u8 one_entry_in_sg = 0;
1082         u8 *src, *dst, *assoc;
1083         __be32 counter = cpu_to_be32(1);
1084         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1085         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
1086         void *aes_ctx = &(ctx->aes_key_expanded);
1087         unsigned long auth_tag_len = crypto_aead_authsize(tfm);
1088         u8 iv_tab[16+AESNI_ALIGN];
1089         u8* iv = (u8 *) PTR_ALIGN((u8 *)iv_tab, AESNI_ALIGN);
1090         struct scatter_walk src_sg_walk;
1091         struct scatter_walk assoc_sg_walk;
1092         struct scatter_walk dst_sg_walk;
1093         unsigned int i;
1094
1095         /* Assuming we are supporting rfc4106 64-bit extended */
1096         /* sequence numbers We need to have the AAD length equal */
1097         /* to 8 or 12 bytes */
1098         if (unlikely(req->assoclen != 8 && req->assoclen != 12))
1099                 return -EINVAL;
1100         /* IV below built */
1101         for (i = 0; i < 4; i++)
1102                 *(iv+i) = ctx->nonce[i];
1103         for (i = 0; i < 8; i++)
1104                 *(iv+4+i) = req->iv[i];
1105         *((__be32 *)(iv+12)) = counter;
1106
1107         if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) {
1108                 one_entry_in_sg = 1;
1109                 scatterwalk_start(&src_sg_walk, req->src);
1110                 scatterwalk_start(&assoc_sg_walk, req->assoc);
1111                 src = scatterwalk_map(&src_sg_walk, 0);
1112                 assoc = scatterwalk_map(&assoc_sg_walk, 0);
1113                 dst = src;
1114                 if (unlikely(req->src != req->dst)) {
1115                         scatterwalk_start(&dst_sg_walk, req->dst);
1116                         dst = scatterwalk_map(&dst_sg_walk, 0);
1117                 }
1118
1119         } else {
1120                 /* Allocate memory for src, dst, assoc */
1121                 src = kmalloc(req->cryptlen + auth_tag_len + req->assoclen,
1122                         GFP_ATOMIC);
1123                 if (unlikely(!src))
1124                         return -ENOMEM;
1125                 assoc = (src + req->cryptlen + auth_tag_len);
1126                 scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0);
1127                 scatterwalk_map_and_copy(assoc, req->assoc, 0,
1128                                         req->assoclen, 0);
1129                 dst = src;
1130         }
1131
1132         aesni_gcm_enc(aes_ctx, dst, src, (unsigned long)req->cryptlen, iv,
1133                 ctx->hash_subkey, assoc, (unsigned long)req->assoclen, dst
1134                 + ((unsigned long)req->cryptlen), auth_tag_len);
1135
1136         /* The authTag (aka the Integrity Check Value) needs to be written
1137          * back to the packet. */
1138         if (one_entry_in_sg) {
1139                 if (unlikely(req->src != req->dst)) {
1140                         scatterwalk_unmap(dst, 0);
1141                         scatterwalk_done(&dst_sg_walk, 0, 0);
1142                 }
1143                 scatterwalk_unmap(src, 0);
1144                 scatterwalk_unmap(assoc, 0);
1145                 scatterwalk_done(&src_sg_walk, 0, 0);
1146                 scatterwalk_done(&assoc_sg_walk, 0, 0);
1147         } else {
1148                 scatterwalk_map_and_copy(dst, req->dst, 0,
1149                         req->cryptlen + auth_tag_len, 1);
1150                 kfree(src);
1151         }
1152         return 0;
1153 }
1154
1155 static int __driver_rfc4106_decrypt(struct aead_request *req)
1156 {
1157         u8 one_entry_in_sg = 0;
1158         u8 *src, *dst, *assoc;
1159         unsigned long tempCipherLen = 0;
1160         __be32 counter = cpu_to_be32(1);
1161         int retval = 0;
1162         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1163         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
1164         void *aes_ctx = &(ctx->aes_key_expanded);
1165         unsigned long auth_tag_len = crypto_aead_authsize(tfm);
1166         u8 iv_and_authTag[32+AESNI_ALIGN];
1167         u8 *iv = (u8 *) PTR_ALIGN((u8 *)iv_and_authTag, AESNI_ALIGN);
1168         u8 *authTag = iv + 16;
1169         struct scatter_walk src_sg_walk;
1170         struct scatter_walk assoc_sg_walk;
1171         struct scatter_walk dst_sg_walk;
1172         unsigned int i;
1173
1174         if (unlikely((req->cryptlen < auth_tag_len) ||
1175                 (req->assoclen != 8 && req->assoclen != 12)))
1176                 return -EINVAL;
1177         /* Assuming we are supporting rfc4106 64-bit extended */
1178         /* sequence numbers We need to have the AAD length */
1179         /* equal to 8 or 12 bytes */
1180
1181         tempCipherLen = (unsigned long)(req->cryptlen - auth_tag_len);
1182         /* IV below built */
1183         for (i = 0; i < 4; i++)
1184                 *(iv+i) = ctx->nonce[i];
1185         for (i = 0; i < 8; i++)
1186                 *(iv+4+i) = req->iv[i];
1187         *((__be32 *)(iv+12)) = counter;
1188
1189         if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) {
1190                 one_entry_in_sg = 1;
1191                 scatterwalk_start(&src_sg_walk, req->src);
1192                 scatterwalk_start(&assoc_sg_walk, req->assoc);
1193                 src = scatterwalk_map(&src_sg_walk, 0);
1194                 assoc = scatterwalk_map(&assoc_sg_walk, 0);
1195                 dst = src;
1196                 if (unlikely(req->src != req->dst)) {
1197                         scatterwalk_start(&dst_sg_walk, req->dst);
1198                         dst = scatterwalk_map(&dst_sg_walk, 0);
1199                 }
1200
1201         } else {
1202                 /* Allocate memory for src, dst, assoc */
1203                 src = kmalloc(req->cryptlen + req->assoclen, GFP_ATOMIC);
1204                 if (!src)
1205                         return -ENOMEM;
1206                 assoc = (src + req->cryptlen + auth_tag_len);
1207                 scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0);
1208                 scatterwalk_map_and_copy(assoc, req->assoc, 0,
1209                         req->assoclen, 0);
1210                 dst = src;
1211         }
1212
1213         aesni_gcm_dec(aes_ctx, dst, src, tempCipherLen, iv,
1214                 ctx->hash_subkey, assoc, (unsigned long)req->assoclen,
1215                 authTag, auth_tag_len);
1216
1217         /* Compare generated tag with passed in tag. */
1218         retval = memcmp(src + tempCipherLen, authTag, auth_tag_len) ?
1219                 -EBADMSG : 0;
1220
1221         if (one_entry_in_sg) {
1222                 if (unlikely(req->src != req->dst)) {
1223                         scatterwalk_unmap(dst, 0);
1224                         scatterwalk_done(&dst_sg_walk, 0, 0);
1225                 }
1226                 scatterwalk_unmap(src, 0);
1227                 scatterwalk_unmap(assoc, 0);
1228                 scatterwalk_done(&src_sg_walk, 0, 0);
1229                 scatterwalk_done(&assoc_sg_walk, 0, 0);
1230         } else {
1231                 scatterwalk_map_and_copy(dst, req->dst, 0, req->cryptlen, 1);
1232                 kfree(src);
1233         }
1234         return retval;
1235 }
1236
1237 static struct crypto_alg __rfc4106_alg = {
1238         .cra_name               = "__gcm-aes-aesni",
1239         .cra_driver_name        = "__driver-gcm-aes-aesni",
1240         .cra_priority           = 0,
1241         .cra_flags              = CRYPTO_ALG_TYPE_AEAD,
1242         .cra_blocksize          = 1,
1243         .cra_ctxsize    = sizeof(struct aesni_rfc4106_gcm_ctx) + AESNI_ALIGN,
1244         .cra_alignmask          = 0,
1245         .cra_type               = &crypto_aead_type,
1246         .cra_module             = THIS_MODULE,
1247         .cra_list               = LIST_HEAD_INIT(__rfc4106_alg.cra_list),
1248         .cra_u = {
1249                 .aead = {
1250                         .encrypt        = __driver_rfc4106_encrypt,
1251                         .decrypt        = __driver_rfc4106_decrypt,
1252                 },
1253         },
1254 };
1255 #endif
1256
1257
1258 static const struct x86_cpu_id aesni_cpu_id[] = {
1259         X86_FEATURE_MATCH(X86_FEATURE_AES),
1260         {}
1261 };
1262 MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id);
1263
1264 static int __init aesni_init(void)
1265 {
1266         int err;
1267
1268         if (!x86_match_cpu(aesni_cpu_id))
1269                 return -ENODEV;
1270
1271         if ((err = crypto_fpu_init()))
1272                 goto fpu_err;
1273         if ((err = crypto_register_alg(&aesni_alg)))
1274                 goto aes_err;
1275         if ((err = crypto_register_alg(&__aesni_alg)))
1276                 goto __aes_err;
1277         if ((err = crypto_register_alg(&blk_ecb_alg)))
1278                 goto blk_ecb_err;
1279         if ((err = crypto_register_alg(&blk_cbc_alg)))
1280                 goto blk_cbc_err;
1281         if ((err = crypto_register_alg(&ablk_ecb_alg)))
1282                 goto ablk_ecb_err;
1283         if ((err = crypto_register_alg(&ablk_cbc_alg)))
1284                 goto ablk_cbc_err;
1285 #ifdef CONFIG_X86_64
1286         if ((err = crypto_register_alg(&blk_ctr_alg)))
1287                 goto blk_ctr_err;
1288         if ((err = crypto_register_alg(&ablk_ctr_alg)))
1289                 goto ablk_ctr_err;
1290         if ((err = crypto_register_alg(&__rfc4106_alg)))
1291                 goto __aead_gcm_err;
1292         if ((err = crypto_register_alg(&rfc4106_alg)))
1293                 goto aead_gcm_err;
1294 #ifdef HAS_CTR
1295         if ((err = crypto_register_alg(&ablk_rfc3686_ctr_alg)))
1296                 goto ablk_rfc3686_ctr_err;
1297 #endif
1298 #endif
1299 #ifdef HAS_LRW
1300         if ((err = crypto_register_alg(&ablk_lrw_alg)))
1301                 goto ablk_lrw_err;
1302 #endif
1303 #ifdef HAS_PCBC
1304         if ((err = crypto_register_alg(&ablk_pcbc_alg)))
1305                 goto ablk_pcbc_err;
1306 #endif
1307 #ifdef HAS_XTS
1308         if ((err = crypto_register_alg(&ablk_xts_alg)))
1309                 goto ablk_xts_err;
1310 #endif
1311         return err;
1312
1313 #ifdef HAS_XTS
1314 ablk_xts_err:
1315 #endif
1316 #ifdef HAS_PCBC
1317         crypto_unregister_alg(&ablk_pcbc_alg);
1318 ablk_pcbc_err:
1319 #endif
1320 #ifdef HAS_LRW
1321         crypto_unregister_alg(&ablk_lrw_alg);
1322 ablk_lrw_err:
1323 #endif
1324 #ifdef CONFIG_X86_64
1325 #ifdef HAS_CTR
1326         crypto_unregister_alg(&ablk_rfc3686_ctr_alg);
1327 ablk_rfc3686_ctr_err:
1328 #endif
1329         crypto_unregister_alg(&rfc4106_alg);
1330 aead_gcm_err:
1331         crypto_unregister_alg(&__rfc4106_alg);
1332 __aead_gcm_err:
1333         crypto_unregister_alg(&ablk_ctr_alg);
1334 ablk_ctr_err:
1335         crypto_unregister_alg(&blk_ctr_alg);
1336 blk_ctr_err:
1337 #endif
1338         crypto_unregister_alg(&ablk_cbc_alg);
1339 ablk_cbc_err:
1340         crypto_unregister_alg(&ablk_ecb_alg);
1341 ablk_ecb_err:
1342         crypto_unregister_alg(&blk_cbc_alg);
1343 blk_cbc_err:
1344         crypto_unregister_alg(&blk_ecb_alg);
1345 blk_ecb_err:
1346         crypto_unregister_alg(&__aesni_alg);
1347 __aes_err:
1348         crypto_unregister_alg(&aesni_alg);
1349 aes_err:
1350 fpu_err:
1351         return err;
1352 }
1353
1354 static void __exit aesni_exit(void)
1355 {
1356 #ifdef HAS_XTS
1357         crypto_unregister_alg(&ablk_xts_alg);
1358 #endif
1359 #ifdef HAS_PCBC
1360         crypto_unregister_alg(&ablk_pcbc_alg);
1361 #endif
1362 #ifdef HAS_LRW
1363         crypto_unregister_alg(&ablk_lrw_alg);
1364 #endif
1365 #ifdef CONFIG_X86_64
1366 #ifdef HAS_CTR
1367         crypto_unregister_alg(&ablk_rfc3686_ctr_alg);
1368 #endif
1369         crypto_unregister_alg(&rfc4106_alg);
1370         crypto_unregister_alg(&__rfc4106_alg);
1371         crypto_unregister_alg(&ablk_ctr_alg);
1372         crypto_unregister_alg(&blk_ctr_alg);
1373 #endif
1374         crypto_unregister_alg(&ablk_cbc_alg);
1375         crypto_unregister_alg(&ablk_ecb_alg);
1376         crypto_unregister_alg(&blk_cbc_alg);
1377         crypto_unregister_alg(&blk_ecb_alg);
1378         crypto_unregister_alg(&__aesni_alg);
1379         crypto_unregister_alg(&aesni_alg);
1380
1381         crypto_fpu_exit();
1382 }
1383
1384 module_init(aesni_init);
1385 module_exit(aesni_exit);
1386
1387 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized");
1388 MODULE_LICENSE("GPL");
1389 MODULE_ALIAS("aes");