crypto: aes-ni - Do not sleep when using the FPU
[linux-2.6.git] / arch / x86 / crypto / aesni-intel_glue.c
1 /*
2  * Support for Intel AES-NI instructions. This file contains glue
3  * code, the real AES implementation is in intel-aes_asm.S.
4  *
5  * Copyright (C) 2008, Intel Corp.
6  *    Author: Huang Ying <ying.huang@intel.com>
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12  */
13
14 #include <linux/hardirq.h>
15 #include <linux/types.h>
16 #include <linux/crypto.h>
17 #include <linux/err.h>
18 #include <crypto/algapi.h>
19 #include <crypto/aes.h>
20 #include <crypto/cryptd.h>
21 #include <asm/i387.h>
22 #include <asm/aes.h>
23
24 #if defined(CONFIG_CRYPTO_CTR) || defined(CONFIG_CRYPTO_CTR_MODULE)
25 #define HAS_CTR
26 #endif
27
28 #if defined(CONFIG_CRYPTO_LRW) || defined(CONFIG_CRYPTO_LRW_MODULE)
29 #define HAS_LRW
30 #endif
31
32 #if defined(CONFIG_CRYPTO_PCBC) || defined(CONFIG_CRYPTO_PCBC_MODULE)
33 #define HAS_PCBC
34 #endif
35
36 #if defined(CONFIG_CRYPTO_XTS) || defined(CONFIG_CRYPTO_XTS_MODULE)
37 #define HAS_XTS
38 #endif
39
40 struct async_aes_ctx {
41         struct cryptd_ablkcipher *cryptd_tfm;
42 };
43
44 #define AESNI_ALIGN     16
45 #define AES_BLOCK_MASK  (~(AES_BLOCK_SIZE-1))
46
47 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
48                              unsigned int key_len);
49 asmlinkage void aesni_enc(struct crypto_aes_ctx *ctx, u8 *out,
50                           const u8 *in);
51 asmlinkage void aesni_dec(struct crypto_aes_ctx *ctx, u8 *out,
52                           const u8 *in);
53 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out,
54                               const u8 *in, unsigned int len);
55 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out,
56                               const u8 *in, unsigned int len);
57 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
58                               const u8 *in, unsigned int len, u8 *iv);
59 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
60                               const u8 *in, unsigned int len, u8 *iv);
61
62 static inline int kernel_fpu_using(void)
63 {
64         if (in_interrupt() && !(read_cr0() & X86_CR0_TS))
65                 return 1;
66         return 0;
67 }
68
69 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
70 {
71         unsigned long addr = (unsigned long)raw_ctx;
72         unsigned long align = AESNI_ALIGN;
73
74         if (align <= crypto_tfm_ctx_alignment())
75                 align = 1;
76         return (struct crypto_aes_ctx *)ALIGN(addr, align);
77 }
78
79 static int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx,
80                               const u8 *in_key, unsigned int key_len)
81 {
82         struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx);
83         u32 *flags = &tfm->crt_flags;
84         int err;
85
86         if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 &&
87             key_len != AES_KEYSIZE_256) {
88                 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
89                 return -EINVAL;
90         }
91
92         if (kernel_fpu_using())
93                 err = crypto_aes_expand_key(ctx, in_key, key_len);
94         else {
95                 kernel_fpu_begin();
96                 err = aesni_set_key(ctx, in_key, key_len);
97                 kernel_fpu_end();
98         }
99
100         return err;
101 }
102
103 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
104                        unsigned int key_len)
105 {
106         return aes_set_key_common(tfm, crypto_tfm_ctx(tfm), in_key, key_len);
107 }
108
109 static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
110 {
111         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
112
113         if (kernel_fpu_using())
114                 crypto_aes_encrypt_x86(ctx, dst, src);
115         else {
116                 kernel_fpu_begin();
117                 aesni_enc(ctx, dst, src);
118                 kernel_fpu_end();
119         }
120 }
121
122 static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
123 {
124         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
125
126         if (kernel_fpu_using())
127                 crypto_aes_decrypt_x86(ctx, dst, src);
128         else {
129                 kernel_fpu_begin();
130                 aesni_dec(ctx, dst, src);
131                 kernel_fpu_end();
132         }
133 }
134
135 static struct crypto_alg aesni_alg = {
136         .cra_name               = "aes",
137         .cra_driver_name        = "aes-aesni",
138         .cra_priority           = 300,
139         .cra_flags              = CRYPTO_ALG_TYPE_CIPHER,
140         .cra_blocksize          = AES_BLOCK_SIZE,
141         .cra_ctxsize            = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
142         .cra_alignmask          = 0,
143         .cra_module             = THIS_MODULE,
144         .cra_list               = LIST_HEAD_INIT(aesni_alg.cra_list),
145         .cra_u  = {
146                 .cipher = {
147                         .cia_min_keysize        = AES_MIN_KEY_SIZE,
148                         .cia_max_keysize        = AES_MAX_KEY_SIZE,
149                         .cia_setkey             = aes_set_key,
150                         .cia_encrypt            = aes_encrypt,
151                         .cia_decrypt            = aes_decrypt
152                 }
153         }
154 };
155
156 static void __aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
157 {
158         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
159
160         aesni_enc(ctx, dst, src);
161 }
162
163 static void __aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
164 {
165         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
166
167         aesni_dec(ctx, dst, src);
168 }
169
170 static struct crypto_alg __aesni_alg = {
171         .cra_name               = "__aes-aesni",
172         .cra_driver_name        = "__driver-aes-aesni",
173         .cra_priority           = 0,
174         .cra_flags              = CRYPTO_ALG_TYPE_CIPHER,
175         .cra_blocksize          = AES_BLOCK_SIZE,
176         .cra_ctxsize            = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
177         .cra_alignmask          = 0,
178         .cra_module             = THIS_MODULE,
179         .cra_list               = LIST_HEAD_INIT(__aesni_alg.cra_list),
180         .cra_u  = {
181                 .cipher = {
182                         .cia_min_keysize        = AES_MIN_KEY_SIZE,
183                         .cia_max_keysize        = AES_MAX_KEY_SIZE,
184                         .cia_setkey             = aes_set_key,
185                         .cia_encrypt            = __aes_encrypt,
186                         .cia_decrypt            = __aes_decrypt
187                 }
188         }
189 };
190
191 static int ecb_encrypt(struct blkcipher_desc *desc,
192                        struct scatterlist *dst, struct scatterlist *src,
193                        unsigned int nbytes)
194 {
195         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
196         struct blkcipher_walk walk;
197         int err;
198
199         blkcipher_walk_init(&walk, dst, src, nbytes);
200         err = blkcipher_walk_virt(desc, &walk);
201         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
202
203         kernel_fpu_begin();
204         while ((nbytes = walk.nbytes)) {
205                 aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
206                               nbytes & AES_BLOCK_MASK);
207                 nbytes &= AES_BLOCK_SIZE - 1;
208                 err = blkcipher_walk_done(desc, &walk, nbytes);
209         }
210         kernel_fpu_end();
211
212         return err;
213 }
214
215 static int ecb_decrypt(struct blkcipher_desc *desc,
216                        struct scatterlist *dst, struct scatterlist *src,
217                        unsigned int nbytes)
218 {
219         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
220         struct blkcipher_walk walk;
221         int err;
222
223         blkcipher_walk_init(&walk, dst, src, nbytes);
224         err = blkcipher_walk_virt(desc, &walk);
225         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
226
227         kernel_fpu_begin();
228         while ((nbytes = walk.nbytes)) {
229                 aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
230                               nbytes & AES_BLOCK_MASK);
231                 nbytes &= AES_BLOCK_SIZE - 1;
232                 err = blkcipher_walk_done(desc, &walk, nbytes);
233         }
234         kernel_fpu_end();
235
236         return err;
237 }
238
239 static struct crypto_alg blk_ecb_alg = {
240         .cra_name               = "__ecb-aes-aesni",
241         .cra_driver_name        = "__driver-ecb-aes-aesni",
242         .cra_priority           = 0,
243         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
244         .cra_blocksize          = AES_BLOCK_SIZE,
245         .cra_ctxsize            = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
246         .cra_alignmask          = 0,
247         .cra_type               = &crypto_blkcipher_type,
248         .cra_module             = THIS_MODULE,
249         .cra_list               = LIST_HEAD_INIT(blk_ecb_alg.cra_list),
250         .cra_u = {
251                 .blkcipher = {
252                         .min_keysize    = AES_MIN_KEY_SIZE,
253                         .max_keysize    = AES_MAX_KEY_SIZE,
254                         .setkey         = aes_set_key,
255                         .encrypt        = ecb_encrypt,
256                         .decrypt        = ecb_decrypt,
257                 },
258         },
259 };
260
261 static int cbc_encrypt(struct blkcipher_desc *desc,
262                        struct scatterlist *dst, struct scatterlist *src,
263                        unsigned int nbytes)
264 {
265         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
266         struct blkcipher_walk walk;
267         int err;
268
269         blkcipher_walk_init(&walk, dst, src, nbytes);
270         err = blkcipher_walk_virt(desc, &walk);
271         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
272
273         kernel_fpu_begin();
274         while ((nbytes = walk.nbytes)) {
275                 aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
276                               nbytes & AES_BLOCK_MASK, walk.iv);
277                 nbytes &= AES_BLOCK_SIZE - 1;
278                 err = blkcipher_walk_done(desc, &walk, nbytes);
279         }
280         kernel_fpu_end();
281
282         return err;
283 }
284
285 static int cbc_decrypt(struct blkcipher_desc *desc,
286                        struct scatterlist *dst, struct scatterlist *src,
287                        unsigned int nbytes)
288 {
289         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
290         struct blkcipher_walk walk;
291         int err;
292
293         blkcipher_walk_init(&walk, dst, src, nbytes);
294         err = blkcipher_walk_virt(desc, &walk);
295         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
296
297         kernel_fpu_begin();
298         while ((nbytes = walk.nbytes)) {
299                 aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
300                               nbytes & AES_BLOCK_MASK, walk.iv);
301                 nbytes &= AES_BLOCK_SIZE - 1;
302                 err = blkcipher_walk_done(desc, &walk, nbytes);
303         }
304         kernel_fpu_end();
305
306         return err;
307 }
308
309 static struct crypto_alg blk_cbc_alg = {
310         .cra_name               = "__cbc-aes-aesni",
311         .cra_driver_name        = "__driver-cbc-aes-aesni",
312         .cra_priority           = 0,
313         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
314         .cra_blocksize          = AES_BLOCK_SIZE,
315         .cra_ctxsize            = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
316         .cra_alignmask          = 0,
317         .cra_type               = &crypto_blkcipher_type,
318         .cra_module             = THIS_MODULE,
319         .cra_list               = LIST_HEAD_INIT(blk_cbc_alg.cra_list),
320         .cra_u = {
321                 .blkcipher = {
322                         .min_keysize    = AES_MIN_KEY_SIZE,
323                         .max_keysize    = AES_MAX_KEY_SIZE,
324                         .setkey         = aes_set_key,
325                         .encrypt        = cbc_encrypt,
326                         .decrypt        = cbc_decrypt,
327                 },
328         },
329 };
330
331 static int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key,
332                         unsigned int key_len)
333 {
334         struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
335         struct crypto_ablkcipher *child = &ctx->cryptd_tfm->base;
336         int err;
337
338         crypto_ablkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
339         crypto_ablkcipher_set_flags(child, crypto_ablkcipher_get_flags(tfm)
340                                     & CRYPTO_TFM_REQ_MASK);
341         err = crypto_ablkcipher_setkey(child, key, key_len);
342         crypto_ablkcipher_set_flags(tfm, crypto_ablkcipher_get_flags(child)
343                                     & CRYPTO_TFM_RES_MASK);
344         return err;
345 }
346
347 static int ablk_encrypt(struct ablkcipher_request *req)
348 {
349         struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
350         struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
351
352         if (kernel_fpu_using()) {
353                 struct ablkcipher_request *cryptd_req =
354                         ablkcipher_request_ctx(req);
355                 memcpy(cryptd_req, req, sizeof(*req));
356                 ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
357                 return crypto_ablkcipher_encrypt(cryptd_req);
358         } else {
359                 struct blkcipher_desc desc;
360                 desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm);
361                 desc.info = req->info;
362                 desc.flags = 0;
363                 return crypto_blkcipher_crt(desc.tfm)->encrypt(
364                         &desc, req->dst, req->src, req->nbytes);
365         }
366 }
367
368 static int ablk_decrypt(struct ablkcipher_request *req)
369 {
370         struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
371         struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
372
373         if (kernel_fpu_using()) {
374                 struct ablkcipher_request *cryptd_req =
375                         ablkcipher_request_ctx(req);
376                 memcpy(cryptd_req, req, sizeof(*req));
377                 ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
378                 return crypto_ablkcipher_decrypt(cryptd_req);
379         } else {
380                 struct blkcipher_desc desc;
381                 desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm);
382                 desc.info = req->info;
383                 desc.flags = 0;
384                 return crypto_blkcipher_crt(desc.tfm)->decrypt(
385                         &desc, req->dst, req->src, req->nbytes);
386         }
387 }
388
389 static void ablk_exit(struct crypto_tfm *tfm)
390 {
391         struct async_aes_ctx *ctx = crypto_tfm_ctx(tfm);
392
393         cryptd_free_ablkcipher(ctx->cryptd_tfm);
394 }
395
396 static void ablk_init_common(struct crypto_tfm *tfm,
397                              struct cryptd_ablkcipher *cryptd_tfm)
398 {
399         struct async_aes_ctx *ctx = crypto_tfm_ctx(tfm);
400
401         ctx->cryptd_tfm = cryptd_tfm;
402         tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request) +
403                 crypto_ablkcipher_reqsize(&cryptd_tfm->base);
404 }
405
406 static int ablk_ecb_init(struct crypto_tfm *tfm)
407 {
408         struct cryptd_ablkcipher *cryptd_tfm;
409
410         cryptd_tfm = cryptd_alloc_ablkcipher("__driver-ecb-aes-aesni", 0, 0);
411         if (IS_ERR(cryptd_tfm))
412                 return PTR_ERR(cryptd_tfm);
413         ablk_init_common(tfm, cryptd_tfm);
414         return 0;
415 }
416
417 static struct crypto_alg ablk_ecb_alg = {
418         .cra_name               = "ecb(aes)",
419         .cra_driver_name        = "ecb-aes-aesni",
420         .cra_priority           = 400,
421         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
422         .cra_blocksize          = AES_BLOCK_SIZE,
423         .cra_ctxsize            = sizeof(struct async_aes_ctx),
424         .cra_alignmask          = 0,
425         .cra_type               = &crypto_ablkcipher_type,
426         .cra_module             = THIS_MODULE,
427         .cra_list               = LIST_HEAD_INIT(ablk_ecb_alg.cra_list),
428         .cra_init               = ablk_ecb_init,
429         .cra_exit               = ablk_exit,
430         .cra_u = {
431                 .ablkcipher = {
432                         .min_keysize    = AES_MIN_KEY_SIZE,
433                         .max_keysize    = AES_MAX_KEY_SIZE,
434                         .setkey         = ablk_set_key,
435                         .encrypt        = ablk_encrypt,
436                         .decrypt        = ablk_decrypt,
437                 },
438         },
439 };
440
441 static int ablk_cbc_init(struct crypto_tfm *tfm)
442 {
443         struct cryptd_ablkcipher *cryptd_tfm;
444
445         cryptd_tfm = cryptd_alloc_ablkcipher("__driver-cbc-aes-aesni", 0, 0);
446         if (IS_ERR(cryptd_tfm))
447                 return PTR_ERR(cryptd_tfm);
448         ablk_init_common(tfm, cryptd_tfm);
449         return 0;
450 }
451
452 static struct crypto_alg ablk_cbc_alg = {
453         .cra_name               = "cbc(aes)",
454         .cra_driver_name        = "cbc-aes-aesni",
455         .cra_priority           = 400,
456         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
457         .cra_blocksize          = AES_BLOCK_SIZE,
458         .cra_ctxsize            = sizeof(struct async_aes_ctx),
459         .cra_alignmask          = 0,
460         .cra_type               = &crypto_ablkcipher_type,
461         .cra_module             = THIS_MODULE,
462         .cra_list               = LIST_HEAD_INIT(ablk_cbc_alg.cra_list),
463         .cra_init               = ablk_cbc_init,
464         .cra_exit               = ablk_exit,
465         .cra_u = {
466                 .ablkcipher = {
467                         .min_keysize    = AES_MIN_KEY_SIZE,
468                         .max_keysize    = AES_MAX_KEY_SIZE,
469                         .ivsize         = AES_BLOCK_SIZE,
470                         .setkey         = ablk_set_key,
471                         .encrypt        = ablk_encrypt,
472                         .decrypt        = ablk_decrypt,
473                 },
474         },
475 };
476
477 #ifdef HAS_CTR
478 static int ablk_ctr_init(struct crypto_tfm *tfm)
479 {
480         struct cryptd_ablkcipher *cryptd_tfm;
481
482         cryptd_tfm = cryptd_alloc_ablkcipher("fpu(ctr(__driver-aes-aesni))",
483                                              0, 0);
484         if (IS_ERR(cryptd_tfm))
485                 return PTR_ERR(cryptd_tfm);
486         ablk_init_common(tfm, cryptd_tfm);
487         return 0;
488 }
489
490 static struct crypto_alg ablk_ctr_alg = {
491         .cra_name               = "ctr(aes)",
492         .cra_driver_name        = "ctr-aes-aesni",
493         .cra_priority           = 400,
494         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
495         .cra_blocksize          = 1,
496         .cra_ctxsize            = sizeof(struct async_aes_ctx),
497         .cra_alignmask          = 0,
498         .cra_type               = &crypto_ablkcipher_type,
499         .cra_module             = THIS_MODULE,
500         .cra_list               = LIST_HEAD_INIT(ablk_ctr_alg.cra_list),
501         .cra_init               = ablk_ctr_init,
502         .cra_exit               = ablk_exit,
503         .cra_u = {
504                 .ablkcipher = {
505                         .min_keysize    = AES_MIN_KEY_SIZE,
506                         .max_keysize    = AES_MAX_KEY_SIZE,
507                         .ivsize         = AES_BLOCK_SIZE,
508                         .setkey         = ablk_set_key,
509                         .encrypt        = ablk_encrypt,
510                         .decrypt        = ablk_decrypt,
511                         .geniv          = "chainiv",
512                 },
513         },
514 };
515 #endif
516
517 #ifdef HAS_LRW
518 static int ablk_lrw_init(struct crypto_tfm *tfm)
519 {
520         struct cryptd_ablkcipher *cryptd_tfm;
521
522         cryptd_tfm = cryptd_alloc_ablkcipher("fpu(lrw(__driver-aes-aesni))",
523                                              0, 0);
524         if (IS_ERR(cryptd_tfm))
525                 return PTR_ERR(cryptd_tfm);
526         ablk_init_common(tfm, cryptd_tfm);
527         return 0;
528 }
529
530 static struct crypto_alg ablk_lrw_alg = {
531         .cra_name               = "lrw(aes)",
532         .cra_driver_name        = "lrw-aes-aesni",
533         .cra_priority           = 400,
534         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
535         .cra_blocksize          = AES_BLOCK_SIZE,
536         .cra_ctxsize            = sizeof(struct async_aes_ctx),
537         .cra_alignmask          = 0,
538         .cra_type               = &crypto_ablkcipher_type,
539         .cra_module             = THIS_MODULE,
540         .cra_list               = LIST_HEAD_INIT(ablk_lrw_alg.cra_list),
541         .cra_init               = ablk_lrw_init,
542         .cra_exit               = ablk_exit,
543         .cra_u = {
544                 .ablkcipher = {
545                         .min_keysize    = AES_MIN_KEY_SIZE + AES_BLOCK_SIZE,
546                         .max_keysize    = AES_MAX_KEY_SIZE + AES_BLOCK_SIZE,
547                         .ivsize         = AES_BLOCK_SIZE,
548                         .setkey         = ablk_set_key,
549                         .encrypt        = ablk_encrypt,
550                         .decrypt        = ablk_decrypt,
551                 },
552         },
553 };
554 #endif
555
556 #ifdef HAS_PCBC
557 static int ablk_pcbc_init(struct crypto_tfm *tfm)
558 {
559         struct cryptd_ablkcipher *cryptd_tfm;
560
561         cryptd_tfm = cryptd_alloc_ablkcipher("fpu(pcbc(__driver-aes-aesni))",
562                                              0, 0);
563         if (IS_ERR(cryptd_tfm))
564                 return PTR_ERR(cryptd_tfm);
565         ablk_init_common(tfm, cryptd_tfm);
566         return 0;
567 }
568
569 static struct crypto_alg ablk_pcbc_alg = {
570         .cra_name               = "pcbc(aes)",
571         .cra_driver_name        = "pcbc-aes-aesni",
572         .cra_priority           = 400,
573         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
574         .cra_blocksize          = AES_BLOCK_SIZE,
575         .cra_ctxsize            = sizeof(struct async_aes_ctx),
576         .cra_alignmask          = 0,
577         .cra_type               = &crypto_ablkcipher_type,
578         .cra_module             = THIS_MODULE,
579         .cra_list               = LIST_HEAD_INIT(ablk_pcbc_alg.cra_list),
580         .cra_init               = ablk_pcbc_init,
581         .cra_exit               = ablk_exit,
582         .cra_u = {
583                 .ablkcipher = {
584                         .min_keysize    = AES_MIN_KEY_SIZE,
585                         .max_keysize    = AES_MAX_KEY_SIZE,
586                         .ivsize         = AES_BLOCK_SIZE,
587                         .setkey         = ablk_set_key,
588                         .encrypt        = ablk_encrypt,
589                         .decrypt        = ablk_decrypt,
590                 },
591         },
592 };
593 #endif
594
595 #ifdef HAS_XTS
596 static int ablk_xts_init(struct crypto_tfm *tfm)
597 {
598         struct cryptd_ablkcipher *cryptd_tfm;
599
600         cryptd_tfm = cryptd_alloc_ablkcipher("fpu(xts(__driver-aes-aesni))",
601                                              0, 0);
602         if (IS_ERR(cryptd_tfm))
603                 return PTR_ERR(cryptd_tfm);
604         ablk_init_common(tfm, cryptd_tfm);
605         return 0;
606 }
607
608 static struct crypto_alg ablk_xts_alg = {
609         .cra_name               = "xts(aes)",
610         .cra_driver_name        = "xts-aes-aesni",
611         .cra_priority           = 400,
612         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
613         .cra_blocksize          = AES_BLOCK_SIZE,
614         .cra_ctxsize            = sizeof(struct async_aes_ctx),
615         .cra_alignmask          = 0,
616         .cra_type               = &crypto_ablkcipher_type,
617         .cra_module             = THIS_MODULE,
618         .cra_list               = LIST_HEAD_INIT(ablk_xts_alg.cra_list),
619         .cra_init               = ablk_xts_init,
620         .cra_exit               = ablk_exit,
621         .cra_u = {
622                 .ablkcipher = {
623                         .min_keysize    = 2 * AES_MIN_KEY_SIZE,
624                         .max_keysize    = 2 * AES_MAX_KEY_SIZE,
625                         .ivsize         = AES_BLOCK_SIZE,
626                         .setkey         = ablk_set_key,
627                         .encrypt        = ablk_encrypt,
628                         .decrypt        = ablk_decrypt,
629                 },
630         },
631 };
632 #endif
633
634 static int __init aesni_init(void)
635 {
636         int err;
637
638         if (!cpu_has_aes) {
639                 printk(KERN_ERR "Intel AES-NI instructions are not detected.\n");
640                 return -ENODEV;
641         }
642         if ((err = crypto_register_alg(&aesni_alg)))
643                 goto aes_err;
644         if ((err = crypto_register_alg(&__aesni_alg)))
645                 goto __aes_err;
646         if ((err = crypto_register_alg(&blk_ecb_alg)))
647                 goto blk_ecb_err;
648         if ((err = crypto_register_alg(&blk_cbc_alg)))
649                 goto blk_cbc_err;
650         if ((err = crypto_register_alg(&ablk_ecb_alg)))
651                 goto ablk_ecb_err;
652         if ((err = crypto_register_alg(&ablk_cbc_alg)))
653                 goto ablk_cbc_err;
654 #ifdef HAS_CTR
655         if ((err = crypto_register_alg(&ablk_ctr_alg)))
656                 goto ablk_ctr_err;
657 #endif
658 #ifdef HAS_LRW
659         if ((err = crypto_register_alg(&ablk_lrw_alg)))
660                 goto ablk_lrw_err;
661 #endif
662 #ifdef HAS_PCBC
663         if ((err = crypto_register_alg(&ablk_pcbc_alg)))
664                 goto ablk_pcbc_err;
665 #endif
666 #ifdef HAS_XTS
667         if ((err = crypto_register_alg(&ablk_xts_alg)))
668                 goto ablk_xts_err;
669 #endif
670
671         return err;
672
673 #ifdef HAS_XTS
674 ablk_xts_err:
675 #endif
676 #ifdef HAS_PCBC
677         crypto_unregister_alg(&ablk_pcbc_alg);
678 ablk_pcbc_err:
679 #endif
680 #ifdef HAS_LRW
681         crypto_unregister_alg(&ablk_lrw_alg);
682 ablk_lrw_err:
683 #endif
684 #ifdef HAS_CTR
685         crypto_unregister_alg(&ablk_ctr_alg);
686 ablk_ctr_err:
687 #endif
688         crypto_unregister_alg(&ablk_cbc_alg);
689 ablk_cbc_err:
690         crypto_unregister_alg(&ablk_ecb_alg);
691 ablk_ecb_err:
692         crypto_unregister_alg(&blk_cbc_alg);
693 blk_cbc_err:
694         crypto_unregister_alg(&blk_ecb_alg);
695 blk_ecb_err:
696         crypto_unregister_alg(&__aesni_alg);
697 __aes_err:
698         crypto_unregister_alg(&aesni_alg);
699 aes_err:
700         return err;
701 }
702
703 static void __exit aesni_exit(void)
704 {
705 #ifdef HAS_XTS
706         crypto_unregister_alg(&ablk_xts_alg);
707 #endif
708 #ifdef HAS_PCBC
709         crypto_unregister_alg(&ablk_pcbc_alg);
710 #endif
711 #ifdef HAS_LRW
712         crypto_unregister_alg(&ablk_lrw_alg);
713 #endif
714 #ifdef HAS_CTR
715         crypto_unregister_alg(&ablk_ctr_alg);
716 #endif
717         crypto_unregister_alg(&ablk_cbc_alg);
718         crypto_unregister_alg(&ablk_ecb_alg);
719         crypto_unregister_alg(&blk_cbc_alg);
720         crypto_unregister_alg(&blk_ecb_alg);
721         crypto_unregister_alg(&__aesni_alg);
722         crypto_unregister_alg(&aesni_alg);
723 }
724
725 module_init(aesni_init);
726 module_exit(aesni_exit);
727
728 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized");
729 MODULE_LICENSE("GPL");
730 MODULE_ALIAS("aes");