crypto: twofish - add 3-way parallel x86_64 assembler implemention
[linux-2.6.git] / arch / x86 / crypto / twofish_glue_3way.c
1 /*
2  * Glue Code for 3-way parallel assembler optimized version of Twofish
3  *
4  * Copyright (c) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
5  *
6  * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by:
7  *   Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
8  * CTR part based on code (crypto/ctr.c) by:
9  *   (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com>
10  *
11  * This program is free software; you can redistribute it and/or modify
12  * it under the terms of the GNU General Public License as published by
13  * the Free Software Foundation; either version 2 of the License, or
14  * (at your option) any later version.
15  *
16  * This program is distributed in the hope that it will be useful,
17  * but WITHOUT ANY WARRANTY; without even the implied warranty of
18  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
19  * GNU General Public License for more details.
20  *
21  * You should have received a copy of the GNU General Public License
22  * along with this program; if not, write to the Free Software
23  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
24  * USA
25  *
26  */
27
28 #include <linux/crypto.h>
29 #include <linux/init.h>
30 #include <linux/module.h>
31 #include <linux/types.h>
32 #include <crypto/algapi.h>
33 #include <crypto/twofish.h>
34 #include <crypto/b128ops.h>
35
36 /* regular block cipher functions from twofish_x86_64 module */
37 asmlinkage void twofish_enc_blk(struct twofish_ctx *ctx, u8 *dst,
38                                 const u8 *src);
39 asmlinkage void twofish_dec_blk(struct twofish_ctx *ctx, u8 *dst,
40                                 const u8 *src);
41
42 /* 3-way parallel cipher functions */
43 asmlinkage void __twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst,
44                                        const u8 *src, bool xor);
45 asmlinkage void twofish_dec_blk_3way(struct twofish_ctx *ctx, u8 *dst,
46                                      const u8 *src);
47
48 static inline void twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst,
49                                         const u8 *src)
50 {
51         __twofish_enc_blk_3way(ctx, dst, src, false);
52 }
53
54 static inline void twofish_enc_blk_xor_3way(struct twofish_ctx *ctx, u8 *dst,
55                                             const u8 *src)
56 {
57         __twofish_enc_blk_3way(ctx, dst, src, true);
58 }
59
60 static int ecb_crypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk,
61                      void (*fn)(struct twofish_ctx *, u8 *, const u8 *),
62                      void (*fn_3way)(struct twofish_ctx *, u8 *, const u8 *))
63 {
64         struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
65         unsigned int bsize = TF_BLOCK_SIZE;
66         unsigned int nbytes;
67         int err;
68
69         err = blkcipher_walk_virt(desc, walk);
70
71         while ((nbytes = walk->nbytes)) {
72                 u8 *wsrc = walk->src.virt.addr;
73                 u8 *wdst = walk->dst.virt.addr;
74
75                 /* Process three block batch */
76                 if (nbytes >= bsize * 3) {
77                         do {
78                                 fn_3way(ctx, wdst, wsrc);
79
80                                 wsrc += bsize * 3;
81                                 wdst += bsize * 3;
82                                 nbytes -= bsize * 3;
83                         } while (nbytes >= bsize * 3);
84
85                         if (nbytes < bsize)
86                                 goto done;
87                 }
88
89                 /* Handle leftovers */
90                 do {
91                         fn(ctx, wdst, wsrc);
92
93                         wsrc += bsize;
94                         wdst += bsize;
95                         nbytes -= bsize;
96                 } while (nbytes >= bsize);
97
98 done:
99                 err = blkcipher_walk_done(desc, walk, nbytes);
100         }
101
102         return err;
103 }
104
105 static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
106                        struct scatterlist *src, unsigned int nbytes)
107 {
108         struct blkcipher_walk walk;
109
110         blkcipher_walk_init(&walk, dst, src, nbytes);
111         return ecb_crypt(desc, &walk, twofish_enc_blk, twofish_enc_blk_3way);
112 }
113
114 static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
115                        struct scatterlist *src, unsigned int nbytes)
116 {
117         struct blkcipher_walk walk;
118
119         blkcipher_walk_init(&walk, dst, src, nbytes);
120         return ecb_crypt(desc, &walk, twofish_dec_blk, twofish_dec_blk_3way);
121 }
122
123 static struct crypto_alg blk_ecb_alg = {
124         .cra_name               = "ecb(twofish)",
125         .cra_driver_name        = "ecb-twofish-3way",
126         .cra_priority           = 300,
127         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
128         .cra_blocksize          = TF_BLOCK_SIZE,
129         .cra_ctxsize            = sizeof(struct twofish_ctx),
130         .cra_alignmask          = 0,
131         .cra_type               = &crypto_blkcipher_type,
132         .cra_module             = THIS_MODULE,
133         .cra_list               = LIST_HEAD_INIT(blk_ecb_alg.cra_list),
134         .cra_u = {
135                 .blkcipher = {
136                         .min_keysize    = TF_MIN_KEY_SIZE,
137                         .max_keysize    = TF_MAX_KEY_SIZE,
138                         .setkey         = twofish_setkey,
139                         .encrypt        = ecb_encrypt,
140                         .decrypt        = ecb_decrypt,
141                 },
142         },
143 };
144
145 static unsigned int __cbc_encrypt(struct blkcipher_desc *desc,
146                                   struct blkcipher_walk *walk)
147 {
148         struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
149         unsigned int bsize = TF_BLOCK_SIZE;
150         unsigned int nbytes = walk->nbytes;
151         u128 *src = (u128 *)walk->src.virt.addr;
152         u128 *dst = (u128 *)walk->dst.virt.addr;
153         u128 *iv = (u128 *)walk->iv;
154
155         do {
156                 u128_xor(dst, src, iv);
157                 twofish_enc_blk(ctx, (u8 *)dst, (u8 *)dst);
158                 iv = dst;
159
160                 src += 1;
161                 dst += 1;
162                 nbytes -= bsize;
163         } while (nbytes >= bsize);
164
165         u128_xor((u128 *)walk->iv, (u128 *)walk->iv, iv);
166         return nbytes;
167 }
168
169 static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
170                        struct scatterlist *src, unsigned int nbytes)
171 {
172         struct blkcipher_walk walk;
173         int err;
174
175         blkcipher_walk_init(&walk, dst, src, nbytes);
176         err = blkcipher_walk_virt(desc, &walk);
177
178         while ((nbytes = walk.nbytes)) {
179                 nbytes = __cbc_encrypt(desc, &walk);
180                 err = blkcipher_walk_done(desc, &walk, nbytes);
181         }
182
183         return err;
184 }
185
186 static unsigned int __cbc_decrypt(struct blkcipher_desc *desc,
187                                   struct blkcipher_walk *walk)
188 {
189         struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
190         unsigned int bsize = TF_BLOCK_SIZE;
191         unsigned int nbytes = walk->nbytes;
192         u128 *src = (u128 *)walk->src.virt.addr;
193         u128 *dst = (u128 *)walk->dst.virt.addr;
194         u128 ivs[3 - 1];
195         u128 last_iv;
196
197         /* Start of the last block. */
198         src += nbytes / bsize - 1;
199         dst += nbytes / bsize - 1;
200
201         last_iv = *src;
202
203         /* Process three block batch */
204         if (nbytes >= bsize * 3) {
205                 do {
206                         nbytes -= bsize * (3 - 1);
207                         src -= 3 - 1;
208                         dst -= 3 - 1;
209
210                         ivs[0] = src[0];
211                         ivs[1] = src[1];
212
213                         twofish_dec_blk_3way(ctx, (u8 *)dst, (u8 *)src);
214
215                         u128_xor(dst + 1, dst + 1, ivs + 0);
216                         u128_xor(dst + 2, dst + 2, ivs + 1);
217
218                         nbytes -= bsize;
219                         if (nbytes < bsize)
220                                 goto done;
221
222                         u128_xor(dst, dst, src - 1);
223                         src -= 1;
224                         dst -= 1;
225                 } while (nbytes >= bsize * 3);
226
227                 if (nbytes < bsize)
228                         goto done;
229         }
230
231         /* Handle leftovers */
232         for (;;) {
233                 twofish_dec_blk(ctx, (u8 *)dst, (u8 *)src);
234
235                 nbytes -= bsize;
236                 if (nbytes < bsize)
237                         break;
238
239                 u128_xor(dst, dst, src - 1);
240                 src -= 1;
241                 dst -= 1;
242         }
243
244 done:
245         u128_xor(dst, dst, (u128 *)walk->iv);
246         *(u128 *)walk->iv = last_iv;
247
248         return nbytes;
249 }
250
251 static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
252                        struct scatterlist *src, unsigned int nbytes)
253 {
254         struct blkcipher_walk walk;
255         int err;
256
257         blkcipher_walk_init(&walk, dst, src, nbytes);
258         err = blkcipher_walk_virt(desc, &walk);
259
260         while ((nbytes = walk.nbytes)) {
261                 nbytes = __cbc_decrypt(desc, &walk);
262                 err = blkcipher_walk_done(desc, &walk, nbytes);
263         }
264
265         return err;
266 }
267
268 static struct crypto_alg blk_cbc_alg = {
269         .cra_name               = "cbc(twofish)",
270         .cra_driver_name        = "cbc-twofish-3way",
271         .cra_priority           = 300,
272         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
273         .cra_blocksize          = TF_BLOCK_SIZE,
274         .cra_ctxsize            = sizeof(struct twofish_ctx),
275         .cra_alignmask          = 0,
276         .cra_type               = &crypto_blkcipher_type,
277         .cra_module             = THIS_MODULE,
278         .cra_list               = LIST_HEAD_INIT(blk_cbc_alg.cra_list),
279         .cra_u = {
280                 .blkcipher = {
281                         .min_keysize    = TF_MIN_KEY_SIZE,
282                         .max_keysize    = TF_MAX_KEY_SIZE,
283                         .ivsize         = TF_BLOCK_SIZE,
284                         .setkey         = twofish_setkey,
285                         .encrypt        = cbc_encrypt,
286                         .decrypt        = cbc_decrypt,
287                 },
288         },
289 };
290
291 static inline void u128_to_be128(be128 *dst, const u128 *src)
292 {
293         dst->a = cpu_to_be64(src->a);
294         dst->b = cpu_to_be64(src->b);
295 }
296
297 static inline void be128_to_u128(u128 *dst, const be128 *src)
298 {
299         dst->a = be64_to_cpu(src->a);
300         dst->b = be64_to_cpu(src->b);
301 }
302
303 static inline void u128_inc(u128 *i)
304 {
305         i->b++;
306         if (!i->b)
307                 i->a++;
308 }
309
310 static void ctr_crypt_final(struct blkcipher_desc *desc,
311                             struct blkcipher_walk *walk)
312 {
313         struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
314         u8 *ctrblk = walk->iv;
315         u8 keystream[TF_BLOCK_SIZE];
316         u8 *src = walk->src.virt.addr;
317         u8 *dst = walk->dst.virt.addr;
318         unsigned int nbytes = walk->nbytes;
319
320         twofish_enc_blk(ctx, keystream, ctrblk);
321         crypto_xor(keystream, src, nbytes);
322         memcpy(dst, keystream, nbytes);
323
324         crypto_inc(ctrblk, TF_BLOCK_SIZE);
325 }
326
327 static unsigned int __ctr_crypt(struct blkcipher_desc *desc,
328                                 struct blkcipher_walk *walk)
329 {
330         struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
331         unsigned int bsize = TF_BLOCK_SIZE;
332         unsigned int nbytes = walk->nbytes;
333         u128 *src = (u128 *)walk->src.virt.addr;
334         u128 *dst = (u128 *)walk->dst.virt.addr;
335         u128 ctrblk;
336         be128 ctrblocks[3];
337
338         be128_to_u128(&ctrblk, (be128 *)walk->iv);
339
340         /* Process three block batch */
341         if (nbytes >= bsize * 3) {
342                 do {
343                         if (dst != src) {
344                                 dst[0] = src[0];
345                                 dst[1] = src[1];
346                                 dst[2] = src[2];
347                         }
348
349                         /* create ctrblks for parallel encrypt */
350                         u128_to_be128(&ctrblocks[0], &ctrblk);
351                         u128_inc(&ctrblk);
352                         u128_to_be128(&ctrblocks[1], &ctrblk);
353                         u128_inc(&ctrblk);
354                         u128_to_be128(&ctrblocks[2], &ctrblk);
355                         u128_inc(&ctrblk);
356
357                         twofish_enc_blk_xor_3way(ctx, (u8 *)dst,
358                                                  (u8 *)ctrblocks);
359
360                         src += 3;
361                         dst += 3;
362                         nbytes -= bsize * 3;
363                 } while (nbytes >= bsize * 3);
364
365                 if (nbytes < bsize)
366                         goto done;
367         }
368
369         /* Handle leftovers */
370         do {
371                 if (dst != src)
372                         *dst = *src;
373
374                 u128_to_be128(&ctrblocks[0], &ctrblk);
375                 u128_inc(&ctrblk);
376
377                 twofish_enc_blk(ctx, (u8 *)ctrblocks, (u8 *)ctrblocks);
378                 u128_xor(dst, dst, (u128 *)ctrblocks);
379
380                 src += 1;
381                 dst += 1;
382                 nbytes -= bsize;
383         } while (nbytes >= bsize);
384
385 done:
386         u128_to_be128((be128 *)walk->iv, &ctrblk);
387         return nbytes;
388 }
389
390 static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
391                      struct scatterlist *src, unsigned int nbytes)
392 {
393         struct blkcipher_walk walk;
394         int err;
395
396         blkcipher_walk_init(&walk, dst, src, nbytes);
397         err = blkcipher_walk_virt_block(desc, &walk, TF_BLOCK_SIZE);
398
399         while ((nbytes = walk.nbytes) >= TF_BLOCK_SIZE) {
400                 nbytes = __ctr_crypt(desc, &walk);
401                 err = blkcipher_walk_done(desc, &walk, nbytes);
402         }
403
404         if (walk.nbytes) {
405                 ctr_crypt_final(desc, &walk);
406                 err = blkcipher_walk_done(desc, &walk, 0);
407         }
408
409         return err;
410 }
411
412 static struct crypto_alg blk_ctr_alg = {
413         .cra_name               = "ctr(twofish)",
414         .cra_driver_name        = "ctr-twofish-3way",
415         .cra_priority           = 300,
416         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
417         .cra_blocksize          = TF_BLOCK_SIZE,
418         .cra_ctxsize            = sizeof(struct twofish_ctx),
419         .cra_alignmask          = 0,
420         .cra_type               = &crypto_blkcipher_type,
421         .cra_module             = THIS_MODULE,
422         .cra_list               = LIST_HEAD_INIT(blk_ctr_alg.cra_list),
423         .cra_u = {
424                 .blkcipher = {
425                         .min_keysize    = TF_MIN_KEY_SIZE,
426                         .max_keysize    = TF_MAX_KEY_SIZE,
427                         .ivsize         = TF_BLOCK_SIZE,
428                         .setkey         = twofish_setkey,
429                         .encrypt        = ctr_crypt,
430                         .decrypt        = ctr_crypt,
431                 },
432         },
433 };
434
435 int __init init(void)
436 {
437         int err;
438
439         err = crypto_register_alg(&blk_ecb_alg);
440         if (err)
441                 goto ecb_err;
442         err = crypto_register_alg(&blk_cbc_alg);
443         if (err)
444                 goto cbc_err;
445         err = crypto_register_alg(&blk_ctr_alg);
446         if (err)
447                 goto ctr_err;
448
449         return 0;
450
451 ctr_err:
452         crypto_unregister_alg(&blk_cbc_alg);
453 cbc_err:
454         crypto_unregister_alg(&blk_ecb_alg);
455 ecb_err:
456         return err;
457 }
458
459 void __exit fini(void)
460 {
461         crypto_unregister_alg(&blk_ctr_alg);
462         crypto_unregister_alg(&blk_cbc_alg);
463         crypto_unregister_alg(&blk_ecb_alg);
464 }
465
466 module_init(init);
467 module_exit(fini);
468
469 MODULE_LICENSE("GPL");
470 MODULE_DESCRIPTION("Twofish Cipher Algorithm, 3-way parallel asm optimized");
471 MODULE_ALIAS("twofish");
472 MODULE_ALIAS("twofish-asm");