crypto: talitos - Avoid consecutive packets going out with same IV
[linux-2.6.git] / drivers / crypto / talitos.c
1 /*
2  * talitos - Freescale Integrated Security Engine (SEC) device driver
3  *
4  * Copyright (c) 2008 Freescale Semiconductor, Inc.
5  *
6  * Scatterlist Crypto API glue code copied from files with the following:
7  * Copyright (c) 2006-2007 Herbert Xu <herbert@gondor.apana.org.au>
8  *
9  * Crypto algorithm registration code copied from hifn driver:
10  * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru>
11  * All rights reserved.
12  *
13  * This program is free software; you can redistribute it and/or modify
14  * it under the terms of the GNU General Public License as published by
15  * the Free Software Foundation; either version 2 of the License, or
16  * (at your option) any later version.
17  *
18  * This program is distributed in the hope that it will be useful,
19  * but WITHOUT ANY WARRANTY; without even the implied warranty of
20  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
21  * GNU General Public License for more details.
22  *
23  * You should have received a copy of the GNU General Public License
24  * along with this program; if not, write to the Free Software
25  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
26  */
27
28 #include <linux/kernel.h>
29 #include <linux/module.h>
30 #include <linux/mod_devicetable.h>
31 #include <linux/device.h>
32 #include <linux/interrupt.h>
33 #include <linux/crypto.h>
34 #include <linux/hw_random.h>
35 #include <linux/of_platform.h>
36 #include <linux/dma-mapping.h>
37 #include <linux/io.h>
38 #include <linux/spinlock.h>
39 #include <linux/rtnetlink.h>
40
41 #include <crypto/algapi.h>
42 #include <crypto/aes.h>
43 #include <crypto/des.h>
44 #include <crypto/sha.h>
45 #include <crypto/aead.h>
46 #include <crypto/authenc.h>
47
48 #include "talitos.h"
49
50 #define TALITOS_TIMEOUT 100000
51 #define TALITOS_MAX_DATA_LEN 65535
52
53 #define DESC_TYPE(desc_hdr) ((be32_to_cpu(desc_hdr) >> 3) & 0x1f)
54 #define PRIMARY_EU(desc_hdr) ((be32_to_cpu(desc_hdr) >> 28) & 0xf)
55 #define SECONDARY_EU(desc_hdr) ((be32_to_cpu(desc_hdr) >> 16) & 0xf)
56
57 /* descriptor pointer entry */
58 struct talitos_ptr {
59         __be16 len;     /* length */
60         u8 j_extent;    /* jump to sg link table and/or extent */
61         u8 eptr;        /* extended address */
62         __be32 ptr;     /* address */
63 };
64
65 /* descriptor */
66 struct talitos_desc {
67         __be32 hdr;                     /* header high bits */
68         __be32 hdr_lo;                  /* header low bits */
69         struct talitos_ptr ptr[7];      /* ptr/len pair array */
70 };
71
72 /**
73  * talitos_request - descriptor submission request
74  * @desc: descriptor pointer (kernel virtual)
75  * @dma_desc: descriptor's physical bus address
76  * @callback: whom to call when descriptor processing is done
77  * @context: caller context (optional)
78  */
79 struct talitos_request {
80         struct talitos_desc *desc;
81         dma_addr_t dma_desc;
82         void (*callback) (struct device *dev, struct talitos_desc *desc,
83                           void *context, int error);
84         void *context;
85 };
86
87 struct talitos_private {
88         struct device *dev;
89         struct of_device *ofdev;
90         void __iomem *reg;
91         int irq;
92
93         /* SEC version geometry (from device tree node) */
94         unsigned int num_channels;
95         unsigned int chfifo_len;
96         unsigned int exec_units;
97         unsigned int desc_types;
98
99         /* SEC Compatibility info */
100         unsigned long features;
101
102         /* next channel to be assigned next incoming descriptor */
103         atomic_t last_chan;
104
105         /* per-channel number of requests pending in channel h/w fifo */
106         atomic_t *submit_count;
107
108         /* per-channel request fifo */
109         struct talitos_request **fifo;
110
111         /*
112          * length of the request fifo
113          * fifo_len is chfifo_len rounded up to next power of 2
114          * so we can use bitwise ops to wrap
115          */
116         unsigned int fifo_len;
117
118         /* per-channel index to next free descriptor request */
119         int *head;
120
121         /* per-channel index to next in-progress/done descriptor request */
122         int *tail;
123
124         /* per-channel request submission (head) and release (tail) locks */
125         spinlock_t *head_lock;
126         spinlock_t *tail_lock;
127
128         /* request callback tasklet */
129         struct tasklet_struct done_task;
130         struct tasklet_struct error_task;
131
132         /* list of registered algorithms */
133         struct list_head alg_list;
134
135         /* hwrng device */
136         struct hwrng rng;
137 };
138
139 /* .features flag */
140 #define TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT 0x00000001
141
142 /*
143  * map virtual single (contiguous) pointer to h/w descriptor pointer
144  */
145 static void map_single_talitos_ptr(struct device *dev,
146                                    struct talitos_ptr *talitos_ptr,
147                                    unsigned short len, void *data,
148                                    unsigned char extent,
149                                    enum dma_data_direction dir)
150 {
151         talitos_ptr->len = cpu_to_be16(len);
152         talitos_ptr->ptr = cpu_to_be32(dma_map_single(dev, data, len, dir));
153         talitos_ptr->j_extent = extent;
154 }
155
156 /*
157  * unmap bus single (contiguous) h/w descriptor pointer
158  */
159 static void unmap_single_talitos_ptr(struct device *dev,
160                                      struct talitos_ptr *talitos_ptr,
161                                      enum dma_data_direction dir)
162 {
163         dma_unmap_single(dev, be32_to_cpu(talitos_ptr->ptr),
164                          be16_to_cpu(talitos_ptr->len), dir);
165 }
166
167 static int reset_channel(struct device *dev, int ch)
168 {
169         struct talitos_private *priv = dev_get_drvdata(dev);
170         unsigned int timeout = TALITOS_TIMEOUT;
171
172         setbits32(priv->reg + TALITOS_CCCR(ch), TALITOS_CCCR_RESET);
173
174         while ((in_be32(priv->reg + TALITOS_CCCR(ch)) & TALITOS_CCCR_RESET)
175                && --timeout)
176                 cpu_relax();
177
178         if (timeout == 0) {
179                 dev_err(dev, "failed to reset channel %d\n", ch);
180                 return -EIO;
181         }
182
183         /* set done writeback and IRQ */
184         setbits32(priv->reg + TALITOS_CCCR_LO(ch), TALITOS_CCCR_LO_CDWE |
185                   TALITOS_CCCR_LO_CDIE);
186
187         return 0;
188 }
189
190 static int reset_device(struct device *dev)
191 {
192         struct talitos_private *priv = dev_get_drvdata(dev);
193         unsigned int timeout = TALITOS_TIMEOUT;
194
195         setbits32(priv->reg + TALITOS_MCR, TALITOS_MCR_SWR);
196
197         while ((in_be32(priv->reg + TALITOS_MCR) & TALITOS_MCR_SWR)
198                && --timeout)
199                 cpu_relax();
200
201         if (timeout == 0) {
202                 dev_err(dev, "failed to reset device\n");
203                 return -EIO;
204         }
205
206         return 0;
207 }
208
209 /*
210  * Reset and initialize the device
211  */
212 static int init_device(struct device *dev)
213 {
214         struct talitos_private *priv = dev_get_drvdata(dev);
215         int ch, err;
216
217         /*
218          * Master reset
219          * errata documentation: warning: certain SEC interrupts
220          * are not fully cleared by writing the MCR:SWR bit,
221          * set bit twice to completely reset
222          */
223         err = reset_device(dev);
224         if (err)
225                 return err;
226
227         err = reset_device(dev);
228         if (err)
229                 return err;
230
231         /* reset channels */
232         for (ch = 0; ch < priv->num_channels; ch++) {
233                 err = reset_channel(dev, ch);
234                 if (err)
235                         return err;
236         }
237
238         /* enable channel done and error interrupts */
239         setbits32(priv->reg + TALITOS_IMR, TALITOS_IMR_INIT);
240         setbits32(priv->reg + TALITOS_IMR_LO, TALITOS_IMR_LO_INIT);
241
242         return 0;
243 }
244
245 /**
246  * talitos_submit - submits a descriptor to the device for processing
247  * @dev:        the SEC device to be used
248  * @desc:       the descriptor to be processed by the device
249  * @callback:   whom to call when processing is complete
250  * @context:    a handle for use by caller (optional)
251  *
252  * desc must contain valid dma-mapped (bus physical) address pointers.
253  * callback must check err and feedback in descriptor header
254  * for device processing status.
255  */
256 static int talitos_submit(struct device *dev, struct talitos_desc *desc,
257                           void (*callback)(struct device *dev,
258                                            struct talitos_desc *desc,
259                                            void *context, int error),
260                           void *context)
261 {
262         struct talitos_private *priv = dev_get_drvdata(dev);
263         struct talitos_request *request;
264         unsigned long flags, ch;
265         int head;
266
267         /* select done notification */
268         desc->hdr |= DESC_HDR_DONE_NOTIFY;
269
270         /* emulate SEC's round-robin channel fifo polling scheme */
271         ch = atomic_inc_return(&priv->last_chan) & (priv->num_channels - 1);
272
273         spin_lock_irqsave(&priv->head_lock[ch], flags);
274
275         if (!atomic_inc_not_zero(&priv->submit_count[ch])) {
276                 /* h/w fifo is full */
277                 spin_unlock_irqrestore(&priv->head_lock[ch], flags);
278                 return -EAGAIN;
279         }
280
281         head = priv->head[ch];
282         request = &priv->fifo[ch][head];
283
284         /* map descriptor and save caller data */
285         request->dma_desc = dma_map_single(dev, desc, sizeof(*desc),
286                                            DMA_BIDIRECTIONAL);
287         request->callback = callback;
288         request->context = context;
289
290         /* increment fifo head */
291         priv->head[ch] = (priv->head[ch] + 1) & (priv->fifo_len - 1);
292
293         smp_wmb();
294         request->desc = desc;
295
296         /* GO! */
297         wmb();
298         out_be32(priv->reg + TALITOS_FF_LO(ch), request->dma_desc);
299
300         spin_unlock_irqrestore(&priv->head_lock[ch], flags);
301
302         return -EINPROGRESS;
303 }
304
305 /*
306  * process what was done, notify callback of error if not
307  */
308 static void flush_channel(struct device *dev, int ch, int error, int reset_ch)
309 {
310         struct talitos_private *priv = dev_get_drvdata(dev);
311         struct talitos_request *request, saved_req;
312         unsigned long flags;
313         int tail, status;
314
315         spin_lock_irqsave(&priv->tail_lock[ch], flags);
316
317         tail = priv->tail[ch];
318         while (priv->fifo[ch][tail].desc) {
319                 request = &priv->fifo[ch][tail];
320
321                 /* descriptors with their done bits set don't get the error */
322                 rmb();
323                 if ((request->desc->hdr & DESC_HDR_DONE) == DESC_HDR_DONE)
324                         status = 0;
325                 else
326                         if (!error)
327                                 break;
328                         else
329                                 status = error;
330
331                 dma_unmap_single(dev, request->dma_desc,
332                         sizeof(struct talitos_desc), DMA_BIDIRECTIONAL);
333
334                 /* copy entries so we can call callback outside lock */
335                 saved_req.desc = request->desc;
336                 saved_req.callback = request->callback;
337                 saved_req.context = request->context;
338
339                 /* release request entry in fifo */
340                 smp_wmb();
341                 request->desc = NULL;
342
343                 /* increment fifo tail */
344                 priv->tail[ch] = (tail + 1) & (priv->fifo_len - 1);
345
346                 spin_unlock_irqrestore(&priv->tail_lock[ch], flags);
347
348                 atomic_dec(&priv->submit_count[ch]);
349
350                 saved_req.callback(dev, saved_req.desc, saved_req.context,
351                                    status);
352                 /* channel may resume processing in single desc error case */
353                 if (error && !reset_ch && status == error)
354                         return;
355                 spin_lock_irqsave(&priv->tail_lock[ch], flags);
356                 tail = priv->tail[ch];
357         }
358
359         spin_unlock_irqrestore(&priv->tail_lock[ch], flags);
360 }
361
362 /*
363  * process completed requests for channels that have done status
364  */
365 static void talitos_done(unsigned long data)
366 {
367         struct device *dev = (struct device *)data;
368         struct talitos_private *priv = dev_get_drvdata(dev);
369         int ch;
370
371         for (ch = 0; ch < priv->num_channels; ch++)
372                 flush_channel(dev, ch, 0, 0);
373 }
374
375 /*
376  * locate current (offending) descriptor
377  */
378 static struct talitos_desc *current_desc(struct device *dev, int ch)
379 {
380         struct talitos_private *priv = dev_get_drvdata(dev);
381         int tail = priv->tail[ch];
382         dma_addr_t cur_desc;
383
384         cur_desc = in_be32(priv->reg + TALITOS_CDPR_LO(ch));
385
386         while (priv->fifo[ch][tail].dma_desc != cur_desc) {
387                 tail = (tail + 1) & (priv->fifo_len - 1);
388                 if (tail == priv->tail[ch]) {
389                         dev_err(dev, "couldn't locate current descriptor\n");
390                         return NULL;
391                 }
392         }
393
394         return priv->fifo[ch][tail].desc;
395 }
396
397 /*
398  * user diagnostics; report root cause of error based on execution unit status
399  */
400 static void report_eu_error(struct device *dev, int ch, struct talitos_desc *desc)
401 {
402         struct talitos_private *priv = dev_get_drvdata(dev);
403         int i;
404
405         switch (desc->hdr & DESC_HDR_SEL0_MASK) {
406         case DESC_HDR_SEL0_AFEU:
407                 dev_err(dev, "AFEUISR 0x%08x_%08x\n",
408                         in_be32(priv->reg + TALITOS_AFEUISR),
409                         in_be32(priv->reg + TALITOS_AFEUISR_LO));
410                 break;
411         case DESC_HDR_SEL0_DEU:
412                 dev_err(dev, "DEUISR 0x%08x_%08x\n",
413                         in_be32(priv->reg + TALITOS_DEUISR),
414                         in_be32(priv->reg + TALITOS_DEUISR_LO));
415                 break;
416         case DESC_HDR_SEL0_MDEUA:
417         case DESC_HDR_SEL0_MDEUB:
418                 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
419                         in_be32(priv->reg + TALITOS_MDEUISR),
420                         in_be32(priv->reg + TALITOS_MDEUISR_LO));
421                 break;
422         case DESC_HDR_SEL0_RNG:
423                 dev_err(dev, "RNGUISR 0x%08x_%08x\n",
424                         in_be32(priv->reg + TALITOS_RNGUISR),
425                         in_be32(priv->reg + TALITOS_RNGUISR_LO));
426                 break;
427         case DESC_HDR_SEL0_PKEU:
428                 dev_err(dev, "PKEUISR 0x%08x_%08x\n",
429                         in_be32(priv->reg + TALITOS_PKEUISR),
430                         in_be32(priv->reg + TALITOS_PKEUISR_LO));
431                 break;
432         case DESC_HDR_SEL0_AESU:
433                 dev_err(dev, "AESUISR 0x%08x_%08x\n",
434                         in_be32(priv->reg + TALITOS_AESUISR),
435                         in_be32(priv->reg + TALITOS_AESUISR_LO));
436                 break;
437         case DESC_HDR_SEL0_CRCU:
438                 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
439                         in_be32(priv->reg + TALITOS_CRCUISR),
440                         in_be32(priv->reg + TALITOS_CRCUISR_LO));
441                 break;
442         case DESC_HDR_SEL0_KEU:
443                 dev_err(dev, "KEUISR 0x%08x_%08x\n",
444                         in_be32(priv->reg + TALITOS_KEUISR),
445                         in_be32(priv->reg + TALITOS_KEUISR_LO));
446                 break;
447         }
448
449         switch (desc->hdr & DESC_HDR_SEL1_MASK) {
450         case DESC_HDR_SEL1_MDEUA:
451         case DESC_HDR_SEL1_MDEUB:
452                 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
453                         in_be32(priv->reg + TALITOS_MDEUISR),
454                         in_be32(priv->reg + TALITOS_MDEUISR_LO));
455                 break;
456         case DESC_HDR_SEL1_CRCU:
457                 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
458                         in_be32(priv->reg + TALITOS_CRCUISR),
459                         in_be32(priv->reg + TALITOS_CRCUISR_LO));
460                 break;
461         }
462
463         for (i = 0; i < 8; i++)
464                 dev_err(dev, "DESCBUF 0x%08x_%08x\n",
465                         in_be32(priv->reg + TALITOS_DESCBUF(ch) + 8*i),
466                         in_be32(priv->reg + TALITOS_DESCBUF_LO(ch) + 8*i));
467 }
468
469 /*
470  * recover from error interrupts
471  */
472 static void talitos_error(unsigned long data)
473 {
474         struct device *dev = (struct device *)data;
475         struct talitos_private *priv = dev_get_drvdata(dev);
476         unsigned int timeout = TALITOS_TIMEOUT;
477         int ch, error, reset_dev = 0, reset_ch = 0;
478         u32 isr, isr_lo, v, v_lo;
479
480         isr = in_be32(priv->reg + TALITOS_ISR);
481         isr_lo = in_be32(priv->reg + TALITOS_ISR_LO);
482
483         for (ch = 0; ch < priv->num_channels; ch++) {
484                 /* skip channels without errors */
485                 if (!(isr & (1 << (ch * 2 + 1))))
486                         continue;
487
488                 error = -EINVAL;
489
490                 v = in_be32(priv->reg + TALITOS_CCPSR(ch));
491                 v_lo = in_be32(priv->reg + TALITOS_CCPSR_LO(ch));
492
493                 if (v_lo & TALITOS_CCPSR_LO_DOF) {
494                         dev_err(dev, "double fetch fifo overflow error\n");
495                         error = -EAGAIN;
496                         reset_ch = 1;
497                 }
498                 if (v_lo & TALITOS_CCPSR_LO_SOF) {
499                         /* h/w dropped descriptor */
500                         dev_err(dev, "single fetch fifo overflow error\n");
501                         error = -EAGAIN;
502                 }
503                 if (v_lo & TALITOS_CCPSR_LO_MDTE)
504                         dev_err(dev, "master data transfer error\n");
505                 if (v_lo & TALITOS_CCPSR_LO_SGDLZ)
506                         dev_err(dev, "s/g data length zero error\n");
507                 if (v_lo & TALITOS_CCPSR_LO_FPZ)
508                         dev_err(dev, "fetch pointer zero error\n");
509                 if (v_lo & TALITOS_CCPSR_LO_IDH)
510                         dev_err(dev, "illegal descriptor header error\n");
511                 if (v_lo & TALITOS_CCPSR_LO_IEU)
512                         dev_err(dev, "invalid execution unit error\n");
513                 if (v_lo & TALITOS_CCPSR_LO_EU)
514                         report_eu_error(dev, ch, current_desc(dev, ch));
515                 if (v_lo & TALITOS_CCPSR_LO_GB)
516                         dev_err(dev, "gather boundary error\n");
517                 if (v_lo & TALITOS_CCPSR_LO_GRL)
518                         dev_err(dev, "gather return/length error\n");
519                 if (v_lo & TALITOS_CCPSR_LO_SB)
520                         dev_err(dev, "scatter boundary error\n");
521                 if (v_lo & TALITOS_CCPSR_LO_SRL)
522                         dev_err(dev, "scatter return/length error\n");
523
524                 flush_channel(dev, ch, error, reset_ch);
525
526                 if (reset_ch) {
527                         reset_channel(dev, ch);
528                 } else {
529                         setbits32(priv->reg + TALITOS_CCCR(ch),
530                                   TALITOS_CCCR_CONT);
531                         setbits32(priv->reg + TALITOS_CCCR_LO(ch), 0);
532                         while ((in_be32(priv->reg + TALITOS_CCCR(ch)) &
533                                TALITOS_CCCR_CONT) && --timeout)
534                                 cpu_relax();
535                         if (timeout == 0) {
536                                 dev_err(dev, "failed to restart channel %d\n",
537                                         ch);
538                                 reset_dev = 1;
539                         }
540                 }
541         }
542         if (reset_dev || isr & ~TALITOS_ISR_CHERR || isr_lo) {
543                 dev_err(dev, "done overflow, internal time out, or rngu error: "
544                         "ISR 0x%08x_%08x\n", isr, isr_lo);
545
546                 /* purge request queues */
547                 for (ch = 0; ch < priv->num_channels; ch++)
548                         flush_channel(dev, ch, -EIO, 1);
549
550                 /* reset and reinitialize the device */
551                 init_device(dev);
552         }
553 }
554
555 static irqreturn_t talitos_interrupt(int irq, void *data)
556 {
557         struct device *dev = data;
558         struct talitos_private *priv = dev_get_drvdata(dev);
559         u32 isr, isr_lo;
560
561         isr = in_be32(priv->reg + TALITOS_ISR);
562         isr_lo = in_be32(priv->reg + TALITOS_ISR_LO);
563
564         /* ack */
565         out_be32(priv->reg + TALITOS_ICR, isr);
566         out_be32(priv->reg + TALITOS_ICR_LO, isr_lo);
567
568         if (unlikely((isr & ~TALITOS_ISR_CHDONE) || isr_lo))
569                 talitos_error((unsigned long)data);
570         else
571                 if (likely(isr & TALITOS_ISR_CHDONE))
572                         tasklet_schedule(&priv->done_task);
573
574         return (isr || isr_lo) ? IRQ_HANDLED : IRQ_NONE;
575 }
576
577 /*
578  * hwrng
579  */
580 static int talitos_rng_data_present(struct hwrng *rng, int wait)
581 {
582         struct device *dev = (struct device *)rng->priv;
583         struct talitos_private *priv = dev_get_drvdata(dev);
584         u32 ofl;
585         int i;
586
587         for (i = 0; i < 20; i++) {
588                 ofl = in_be32(priv->reg + TALITOS_RNGUSR_LO) &
589                       TALITOS_RNGUSR_LO_OFL;
590                 if (ofl || !wait)
591                         break;
592                 udelay(10);
593         }
594
595         return !!ofl;
596 }
597
598 static int talitos_rng_data_read(struct hwrng *rng, u32 *data)
599 {
600         struct device *dev = (struct device *)rng->priv;
601         struct talitos_private *priv = dev_get_drvdata(dev);
602
603         /* rng fifo requires 64-bit accesses */
604         *data = in_be32(priv->reg + TALITOS_RNGU_FIFO);
605         *data = in_be32(priv->reg + TALITOS_RNGU_FIFO_LO);
606
607         return sizeof(u32);
608 }
609
610 static int talitos_rng_init(struct hwrng *rng)
611 {
612         struct device *dev = (struct device *)rng->priv;
613         struct talitos_private *priv = dev_get_drvdata(dev);
614         unsigned int timeout = TALITOS_TIMEOUT;
615
616         setbits32(priv->reg + TALITOS_RNGURCR_LO, TALITOS_RNGURCR_LO_SR);
617         while (!(in_be32(priv->reg + TALITOS_RNGUSR_LO) & TALITOS_RNGUSR_LO_RD)
618                && --timeout)
619                 cpu_relax();
620         if (timeout == 0) {
621                 dev_err(dev, "failed to reset rng hw\n");
622                 return -ENODEV;
623         }
624
625         /* start generating */
626         setbits32(priv->reg + TALITOS_RNGUDSR_LO, 0);
627
628         return 0;
629 }
630
631 static int talitos_register_rng(struct device *dev)
632 {
633         struct talitos_private *priv = dev_get_drvdata(dev);
634
635         priv->rng.name          = dev_driver_string(dev),
636         priv->rng.init          = talitos_rng_init,
637         priv->rng.data_present  = talitos_rng_data_present,
638         priv->rng.data_read     = talitos_rng_data_read,
639         priv->rng.priv          = (unsigned long)dev;
640
641         return hwrng_register(&priv->rng);
642 }
643
644 static void talitos_unregister_rng(struct device *dev)
645 {
646         struct talitos_private *priv = dev_get_drvdata(dev);
647
648         hwrng_unregister(&priv->rng);
649 }
650
651 /*
652  * crypto alg
653  */
654 #define TALITOS_CRA_PRIORITY            3000
655 #define TALITOS_MAX_KEY_SIZE            64
656 #define TALITOS_MAX_IV_LENGTH           16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
657
658 #define MD5_DIGEST_SIZE   16
659
660 struct talitos_ctx {
661         struct device *dev;
662         __be32 desc_hdr_template;
663         u8 key[TALITOS_MAX_KEY_SIZE];
664         u8 iv[TALITOS_MAX_IV_LENGTH];
665         unsigned int keylen;
666         unsigned int enckeylen;
667         unsigned int authkeylen;
668         unsigned int authsize;
669 };
670
671 static int aead_authenc_setauthsize(struct crypto_aead *authenc,
672                                                  unsigned int authsize)
673 {
674         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
675
676         ctx->authsize = authsize;
677
678         return 0;
679 }
680
681 static int aead_authenc_setkey(struct crypto_aead *authenc,
682                                             const u8 *key, unsigned int keylen)
683 {
684         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
685         struct rtattr *rta = (void *)key;
686         struct crypto_authenc_key_param *param;
687         unsigned int authkeylen;
688         unsigned int enckeylen;
689
690         if (!RTA_OK(rta, keylen))
691                 goto badkey;
692
693         if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM)
694                 goto badkey;
695
696         if (RTA_PAYLOAD(rta) < sizeof(*param))
697                 goto badkey;
698
699         param = RTA_DATA(rta);
700         enckeylen = be32_to_cpu(param->enckeylen);
701
702         key += RTA_ALIGN(rta->rta_len);
703         keylen -= RTA_ALIGN(rta->rta_len);
704
705         if (keylen < enckeylen)
706                 goto badkey;
707
708         authkeylen = keylen - enckeylen;
709
710         if (keylen > TALITOS_MAX_KEY_SIZE)
711                 goto badkey;
712
713         memcpy(&ctx->key, key, keylen);
714
715         ctx->keylen = keylen;
716         ctx->enckeylen = enckeylen;
717         ctx->authkeylen = authkeylen;
718
719         return 0;
720
721 badkey:
722         crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN);
723         return -EINVAL;
724 }
725
726 /*
727  * ipsec_esp_edesc - s/w-extended ipsec_esp descriptor
728  * @src_nents: number of segments in input scatterlist
729  * @dst_nents: number of segments in output scatterlist
730  * @dma_len: length of dma mapped link_tbl space
731  * @dma_link_tbl: bus physical address of link_tbl
732  * @desc: h/w descriptor
733  * @link_tbl: input and output h/w link tables (if {src,dst}_nents > 1)
734  *
735  * if decrypting (with authcheck), or either one of src_nents or dst_nents
736  * is greater than 1, an integrity check value is concatenated to the end
737  * of link_tbl data
738  */
739 struct ipsec_esp_edesc {
740         int src_nents;
741         int dst_nents;
742         int dma_len;
743         dma_addr_t dma_link_tbl;
744         struct talitos_desc desc;
745         struct talitos_ptr link_tbl[0];
746 };
747
748 static void ipsec_esp_unmap(struct device *dev,
749                             struct ipsec_esp_edesc *edesc,
750                             struct aead_request *areq)
751 {
752         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[6], DMA_FROM_DEVICE);
753         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[3], DMA_TO_DEVICE);
754         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE);
755         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[0], DMA_TO_DEVICE);
756
757         dma_unmap_sg(dev, areq->assoc, 1, DMA_TO_DEVICE);
758
759         if (areq->src != areq->dst) {
760                 dma_unmap_sg(dev, areq->src, edesc->src_nents ? : 1,
761                              DMA_TO_DEVICE);
762                 dma_unmap_sg(dev, areq->dst, edesc->dst_nents ? : 1,
763                              DMA_FROM_DEVICE);
764         } else {
765                 dma_unmap_sg(dev, areq->src, edesc->src_nents ? : 1,
766                              DMA_BIDIRECTIONAL);
767         }
768
769         if (edesc->dma_len)
770                 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
771                                  DMA_BIDIRECTIONAL);
772 }
773
774 /*
775  * ipsec_esp descriptor callbacks
776  */
777 static void ipsec_esp_encrypt_done(struct device *dev,
778                                    struct talitos_desc *desc, void *context,
779                                    int err)
780 {
781         struct aead_request *areq = context;
782         struct ipsec_esp_edesc *edesc =
783                  container_of(desc, struct ipsec_esp_edesc, desc);
784         struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
785         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
786         struct scatterlist *sg;
787         void *icvdata;
788
789         ipsec_esp_unmap(dev, edesc, areq);
790
791         /* copy the generated ICV to dst */
792         if (edesc->dma_len) {
793                 icvdata = &edesc->link_tbl[edesc->src_nents +
794                                            edesc->dst_nents + 2];
795                 sg = sg_last(areq->dst, edesc->dst_nents);
796                 memcpy((char *)sg_virt(sg) + sg->length - ctx->authsize,
797                        icvdata, ctx->authsize);
798         }
799
800         kfree(edesc);
801
802         aead_request_complete(areq, err);
803 }
804
805 static void ipsec_esp_decrypt_done(struct device *dev,
806                                    struct talitos_desc *desc, void *context,
807                                    int err)
808 {
809         struct aead_request *req = context;
810         struct ipsec_esp_edesc *edesc =
811                  container_of(desc, struct ipsec_esp_edesc, desc);
812         struct crypto_aead *authenc = crypto_aead_reqtfm(req);
813         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
814         struct scatterlist *sg;
815         void *icvdata;
816
817         ipsec_esp_unmap(dev, edesc, req);
818
819         if (!err) {
820                 /* auth check */
821                 if (edesc->dma_len)
822                         icvdata = &edesc->link_tbl[edesc->src_nents +
823                                                    edesc->dst_nents + 2];
824                 else
825                         icvdata = &edesc->link_tbl[0];
826
827                 sg = sg_last(req->dst, edesc->dst_nents ? : 1);
828                 err = memcmp(icvdata, (char *)sg_virt(sg) + sg->length -
829                              ctx->authsize, ctx->authsize) ? -EBADMSG : 0;
830         }
831
832         kfree(edesc);
833
834         aead_request_complete(req, err);
835 }
836
837 /*
838  * convert scatterlist to SEC h/w link table format
839  * stop at cryptlen bytes
840  */
841 static int sg_to_link_tbl(struct scatterlist *sg, int sg_count,
842                            int cryptlen, struct talitos_ptr *link_tbl_ptr)
843 {
844         int n_sg = sg_count;
845
846         while (n_sg--) {
847                 link_tbl_ptr->ptr = cpu_to_be32(sg_dma_address(sg));
848                 link_tbl_ptr->len = cpu_to_be16(sg_dma_len(sg));
849                 link_tbl_ptr->j_extent = 0;
850                 link_tbl_ptr++;
851                 cryptlen -= sg_dma_len(sg);
852                 sg = sg_next(sg);
853         }
854
855         /* adjust (decrease) last one (or two) entry's len to cryptlen */
856         link_tbl_ptr--;
857         while (be16_to_cpu(link_tbl_ptr->len) <= (-cryptlen)) {
858                 /* Empty this entry, and move to previous one */
859                 cryptlen += be16_to_cpu(link_tbl_ptr->len);
860                 link_tbl_ptr->len = 0;
861                 sg_count--;
862                 link_tbl_ptr--;
863         }
864         link_tbl_ptr->len = cpu_to_be16(be16_to_cpu(link_tbl_ptr->len)
865                                         + cryptlen);
866
867         /* tag end of link table */
868         link_tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN;
869
870         return sg_count;
871 }
872
873 /*
874  * fill in and submit ipsec_esp descriptor
875  */
876 static int ipsec_esp(struct ipsec_esp_edesc *edesc, struct aead_request *areq,
877                      u8 *giv, u64 seq,
878                      void (*callback) (struct device *dev,
879                                        struct talitos_desc *desc,
880                                        void *context, int error))
881 {
882         struct crypto_aead *aead = crypto_aead_reqtfm(areq);
883         struct talitos_ctx *ctx = crypto_aead_ctx(aead);
884         struct device *dev = ctx->dev;
885         struct talitos_desc *desc = &edesc->desc;
886         unsigned int cryptlen = areq->cryptlen;
887         unsigned int authsize = ctx->authsize;
888         unsigned int ivsize;
889         int sg_count, ret;
890
891         /* hmac key */
892         map_single_talitos_ptr(dev, &desc->ptr[0], ctx->authkeylen, &ctx->key,
893                                0, DMA_TO_DEVICE);
894         /* hmac data */
895         map_single_talitos_ptr(dev, &desc->ptr[1], sg_virt(areq->src) -
896                                sg_virt(areq->assoc), sg_virt(areq->assoc), 0,
897                                DMA_TO_DEVICE);
898         /* cipher iv */
899         ivsize = crypto_aead_ivsize(aead);
900         map_single_talitos_ptr(dev, &desc->ptr[2], ivsize, giv ?: areq->iv, 0,
901                                DMA_TO_DEVICE);
902
903         /* cipher key */
904         map_single_talitos_ptr(dev, &desc->ptr[3], ctx->enckeylen,
905                                (char *)&ctx->key + ctx->authkeylen, 0,
906                                DMA_TO_DEVICE);
907
908         /*
909          * cipher in
910          * map and adjust cipher len to aead request cryptlen.
911          * extent is bytes of HMAC postpended to ciphertext,
912          * typically 12 for ipsec
913          */
914         desc->ptr[4].len = cpu_to_be16(cryptlen);
915         desc->ptr[4].j_extent = authsize;
916
917         if (areq->src == areq->dst)
918                 sg_count = dma_map_sg(dev, areq->src, edesc->src_nents ? : 1,
919                                       DMA_BIDIRECTIONAL);
920         else
921                 sg_count = dma_map_sg(dev, areq->src, edesc->src_nents ? : 1,
922                                       DMA_TO_DEVICE);
923
924         if (sg_count == 1) {
925                 desc->ptr[4].ptr = cpu_to_be32(sg_dma_address(areq->src));
926         } else {
927                 sg_count = sg_to_link_tbl(areq->src, sg_count, cryptlen,
928                                           &edesc->link_tbl[0]);
929                 if (sg_count > 1) {
930                         struct talitos_ptr *link_tbl_ptr =
931                                 &edesc->link_tbl[sg_count-1];
932                         struct scatterlist *sg;
933                         struct talitos_private *priv = dev_get_drvdata(dev);
934
935                         desc->ptr[4].j_extent |= DESC_PTR_LNKTBL_JUMP;
936                         desc->ptr[4].ptr = cpu_to_be32(edesc->dma_link_tbl);
937                         dma_sync_single_for_device(ctx->dev, edesc->dma_link_tbl,
938                                                    edesc->dma_len, DMA_BIDIRECTIONAL);
939                         /* If necessary for this SEC revision,
940                          * add a link table entry for ICV.
941                          */
942                         if ((priv->features &
943                              TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT) &&
944                             (edesc->desc.hdr & DESC_HDR_MODE0_ENCRYPT) == 0) {
945                                 link_tbl_ptr->j_extent = 0;
946                                 link_tbl_ptr++;
947                                 link_tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN;
948                                 link_tbl_ptr->len = cpu_to_be16(authsize);
949                                 sg = sg_last(areq->src, edesc->src_nents ? : 1);
950                                 link_tbl_ptr->ptr = cpu_to_be32(
951                                                 (char *)sg_dma_address(sg)
952                                                 + sg->length - authsize);
953                         }
954                 } else {
955                         /* Only one segment now, so no link tbl needed */
956                         desc->ptr[4].ptr = cpu_to_be32(sg_dma_address(areq->src));
957                 }
958         }
959
960         /* cipher out */
961         desc->ptr[5].len = cpu_to_be16(cryptlen);
962         desc->ptr[5].j_extent = authsize;
963
964         if (areq->src != areq->dst) {
965                 sg_count = dma_map_sg(dev, areq->dst, edesc->dst_nents ? : 1,
966                                       DMA_FROM_DEVICE);
967         }
968
969         if (sg_count == 1) {
970                 desc->ptr[5].ptr = cpu_to_be32(sg_dma_address(areq->dst));
971         } else {
972                 struct talitos_ptr *link_tbl_ptr =
973                         &edesc->link_tbl[edesc->src_nents + 1];
974
975                 desc->ptr[5].ptr = cpu_to_be32((struct talitos_ptr *)
976                                                edesc->dma_link_tbl +
977                                                edesc->src_nents + 1);
978                 if (areq->src == areq->dst) {
979                         memcpy(link_tbl_ptr, &edesc->link_tbl[0],
980                                edesc->src_nents * sizeof(struct talitos_ptr));
981                 } else {
982                         sg_count = sg_to_link_tbl(areq->dst, sg_count, cryptlen,
983                                                   link_tbl_ptr);
984                 }
985                 /* Add an entry to the link table for ICV data */
986                 link_tbl_ptr += sg_count - 1;
987                 link_tbl_ptr->j_extent = 0;
988                 sg_count++;
989                 link_tbl_ptr++;
990                 link_tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN;
991                 link_tbl_ptr->len = cpu_to_be16(authsize);
992
993                 /* icv data follows link tables */
994                 link_tbl_ptr->ptr = cpu_to_be32((struct talitos_ptr *)
995                                                 edesc->dma_link_tbl +
996                                                 edesc->src_nents +
997                                                 edesc->dst_nents + 2);
998
999                 desc->ptr[5].j_extent |= DESC_PTR_LNKTBL_JUMP;
1000                 dma_sync_single_for_device(ctx->dev, edesc->dma_link_tbl,
1001                                            edesc->dma_len, DMA_BIDIRECTIONAL);
1002         }
1003
1004         /* iv out */
1005         map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv, 0,
1006                                DMA_FROM_DEVICE);
1007
1008         ret = talitos_submit(dev, desc, callback, areq);
1009         if (ret != -EINPROGRESS) {
1010                 ipsec_esp_unmap(dev, edesc, areq);
1011                 kfree(edesc);
1012         }
1013         return ret;
1014 }
1015
1016
1017 /*
1018  * derive number of elements in scatterlist
1019  */
1020 static int sg_count(struct scatterlist *sg_list, int nbytes)
1021 {
1022         struct scatterlist *sg = sg_list;
1023         int sg_nents = 0;
1024
1025         while (nbytes) {
1026                 sg_nents++;
1027                 nbytes -= sg->length;
1028                 sg = sg_next(sg);
1029         }
1030
1031         return sg_nents;
1032 }
1033
1034 /*
1035  * allocate and map the ipsec_esp extended descriptor
1036  */
1037 static struct ipsec_esp_edesc *ipsec_esp_edesc_alloc(struct aead_request *areq,
1038                                                      int icv_stashing)
1039 {
1040         struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
1041         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1042         struct ipsec_esp_edesc *edesc;
1043         int src_nents, dst_nents, alloc_len, dma_len;
1044         gfp_t flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
1045                       GFP_ATOMIC;
1046
1047         if (areq->cryptlen + ctx->authsize > TALITOS_MAX_DATA_LEN) {
1048                 dev_err(ctx->dev, "cryptlen exceeds h/w max limit\n");
1049                 return ERR_PTR(-EINVAL);
1050         }
1051
1052         src_nents = sg_count(areq->src, areq->cryptlen + ctx->authsize);
1053         src_nents = (src_nents == 1) ? 0 : src_nents;
1054
1055         if (areq->dst == areq->src) {
1056                 dst_nents = src_nents;
1057         } else {
1058                 dst_nents = sg_count(areq->dst, areq->cryptlen + ctx->authsize);
1059                 dst_nents = (dst_nents == 1) ? 0 : dst_nents;
1060         }
1061
1062         /*
1063          * allocate space for base edesc plus the link tables,
1064          * allowing for two separate entries for ICV and generated ICV (+ 2),
1065          * and the ICV data itself
1066          */
1067         alloc_len = sizeof(struct ipsec_esp_edesc);
1068         if (src_nents || dst_nents) {
1069                 dma_len = (src_nents + dst_nents + 2) *
1070                                  sizeof(struct talitos_ptr) + ctx->authsize;
1071                 alloc_len += dma_len;
1072         } else {
1073                 dma_len = 0;
1074                 alloc_len += icv_stashing ? ctx->authsize : 0;
1075         }
1076
1077         edesc = kmalloc(alloc_len, GFP_DMA | flags);
1078         if (!edesc) {
1079                 dev_err(ctx->dev, "could not allocate edescriptor\n");
1080                 return ERR_PTR(-ENOMEM);
1081         }
1082
1083         edesc->src_nents = src_nents;
1084         edesc->dst_nents = dst_nents;
1085         edesc->dma_len = dma_len;
1086         edesc->dma_link_tbl = dma_map_single(ctx->dev, &edesc->link_tbl[0],
1087                                              edesc->dma_len, DMA_BIDIRECTIONAL);
1088
1089         return edesc;
1090 }
1091
1092 static int aead_authenc_encrypt(struct aead_request *req)
1093 {
1094         struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1095         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1096         struct ipsec_esp_edesc *edesc;
1097
1098         /* allocate extended descriptor */
1099         edesc = ipsec_esp_edesc_alloc(req, 0);
1100         if (IS_ERR(edesc))
1101                 return PTR_ERR(edesc);
1102
1103         /* set encrypt */
1104         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1105
1106         return ipsec_esp(edesc, req, NULL, 0, ipsec_esp_encrypt_done);
1107 }
1108
1109 static int aead_authenc_decrypt(struct aead_request *req)
1110 {
1111         struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1112         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1113         unsigned int authsize = ctx->authsize;
1114         struct ipsec_esp_edesc *edesc;
1115         struct scatterlist *sg;
1116         void *icvdata;
1117
1118         req->cryptlen -= authsize;
1119
1120         /* allocate extended descriptor */
1121         edesc = ipsec_esp_edesc_alloc(req, 1);
1122         if (IS_ERR(edesc))
1123                 return PTR_ERR(edesc);
1124
1125         /* stash incoming ICV for later cmp with ICV generated by the h/w */
1126         if (edesc->dma_len)
1127                 icvdata = &edesc->link_tbl[edesc->src_nents +
1128                                            edesc->dst_nents + 2];
1129         else
1130                 icvdata = &edesc->link_tbl[0];
1131
1132         sg = sg_last(req->src, edesc->src_nents ? : 1);
1133
1134         memcpy(icvdata, (char *)sg_virt(sg) + sg->length - ctx->authsize,
1135                ctx->authsize);
1136
1137         /* decrypt */
1138         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1139
1140         return ipsec_esp(edesc, req, NULL, 0, ipsec_esp_decrypt_done);
1141 }
1142
1143 static int aead_authenc_givencrypt(
1144         struct aead_givcrypt_request *req)
1145 {
1146         struct aead_request *areq = &req->areq;
1147         struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
1148         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1149         struct ipsec_esp_edesc *edesc;
1150
1151         /* allocate extended descriptor */
1152         edesc = ipsec_esp_edesc_alloc(areq, 0);
1153         if (IS_ERR(edesc))
1154                 return PTR_ERR(edesc);
1155
1156         /* set encrypt */
1157         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1158
1159         memcpy(req->giv, ctx->iv, crypto_aead_ivsize(authenc));
1160         /* avoid consecutive packets going out with same IV */
1161         *(__be64 *)req->giv ^= cpu_to_be64(req->seq);
1162
1163         return ipsec_esp(edesc, areq, req->giv, req->seq,
1164                          ipsec_esp_encrypt_done);
1165 }
1166
1167 struct talitos_alg_template {
1168         char name[CRYPTO_MAX_ALG_NAME];
1169         char driver_name[CRYPTO_MAX_ALG_NAME];
1170         unsigned int blocksize;
1171         struct aead_alg aead;
1172         struct device *dev;
1173         __be32 desc_hdr_template;
1174 };
1175
1176 static struct talitos_alg_template driver_algs[] = {
1177         /* single-pass ipsec_esp descriptor */
1178         {
1179                 .name = "authenc(hmac(sha1),cbc(aes))",
1180                 .driver_name = "authenc-hmac-sha1-cbc-aes-talitos",
1181                 .blocksize = AES_BLOCK_SIZE,
1182                 .aead = {
1183                         .setkey = aead_authenc_setkey,
1184                         .setauthsize = aead_authenc_setauthsize,
1185                         .encrypt = aead_authenc_encrypt,
1186                         .decrypt = aead_authenc_decrypt,
1187                         .givencrypt = aead_authenc_givencrypt,
1188                         .geniv = "<built-in>",
1189                         .ivsize = AES_BLOCK_SIZE,
1190                         .maxauthsize = SHA1_DIGEST_SIZE,
1191                         },
1192                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
1193                                      DESC_HDR_SEL0_AESU |
1194                                      DESC_HDR_MODE0_AESU_CBC |
1195                                      DESC_HDR_SEL1_MDEUA |
1196                                      DESC_HDR_MODE1_MDEU_INIT |
1197                                      DESC_HDR_MODE1_MDEU_PAD |
1198                                      DESC_HDR_MODE1_MDEU_SHA1_HMAC,
1199         },
1200         {
1201                 .name = "authenc(hmac(sha1),cbc(des3_ede))",
1202                 .driver_name = "authenc-hmac-sha1-cbc-3des-talitos",
1203                 .blocksize = DES3_EDE_BLOCK_SIZE,
1204                 .aead = {
1205                         .setkey = aead_authenc_setkey,
1206                         .setauthsize = aead_authenc_setauthsize,
1207                         .encrypt = aead_authenc_encrypt,
1208                         .decrypt = aead_authenc_decrypt,
1209                         .givencrypt = aead_authenc_givencrypt,
1210                         .geniv = "<built-in>",
1211                         .ivsize = DES3_EDE_BLOCK_SIZE,
1212                         .maxauthsize = SHA1_DIGEST_SIZE,
1213                         },
1214                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
1215                                      DESC_HDR_SEL0_DEU |
1216                                      DESC_HDR_MODE0_DEU_CBC |
1217                                      DESC_HDR_MODE0_DEU_3DES |
1218                                      DESC_HDR_SEL1_MDEUA |
1219                                      DESC_HDR_MODE1_MDEU_INIT |
1220                                      DESC_HDR_MODE1_MDEU_PAD |
1221                                      DESC_HDR_MODE1_MDEU_SHA1_HMAC,
1222         },
1223         {
1224                 .name = "authenc(hmac(sha256),cbc(aes))",
1225                 .driver_name = "authenc-hmac-sha256-cbc-aes-talitos",
1226                 .blocksize = AES_BLOCK_SIZE,
1227                 .aead = {
1228                         .setkey = aead_authenc_setkey,
1229                         .setauthsize = aead_authenc_setauthsize,
1230                         .encrypt = aead_authenc_encrypt,
1231                         .decrypt = aead_authenc_decrypt,
1232                         .givencrypt = aead_authenc_givencrypt,
1233                         .geniv = "<built-in>",
1234                         .ivsize = AES_BLOCK_SIZE,
1235                         .maxauthsize = SHA256_DIGEST_SIZE,
1236                         },
1237                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
1238                                      DESC_HDR_SEL0_AESU |
1239                                      DESC_HDR_MODE0_AESU_CBC |
1240                                      DESC_HDR_SEL1_MDEUA |
1241                                      DESC_HDR_MODE1_MDEU_INIT |
1242                                      DESC_HDR_MODE1_MDEU_PAD |
1243                                      DESC_HDR_MODE1_MDEU_SHA256_HMAC,
1244         },
1245         {
1246                 .name = "authenc(hmac(sha256),cbc(des3_ede))",
1247                 .driver_name = "authenc-hmac-sha256-cbc-3des-talitos",
1248                 .blocksize = DES3_EDE_BLOCK_SIZE,
1249                 .aead = {
1250                         .setkey = aead_authenc_setkey,
1251                         .setauthsize = aead_authenc_setauthsize,
1252                         .encrypt = aead_authenc_encrypt,
1253                         .decrypt = aead_authenc_decrypt,
1254                         .givencrypt = aead_authenc_givencrypt,
1255                         .geniv = "<built-in>",
1256                         .ivsize = DES3_EDE_BLOCK_SIZE,
1257                         .maxauthsize = SHA256_DIGEST_SIZE,
1258                         },
1259                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
1260                                      DESC_HDR_SEL0_DEU |
1261                                      DESC_HDR_MODE0_DEU_CBC |
1262                                      DESC_HDR_MODE0_DEU_3DES |
1263                                      DESC_HDR_SEL1_MDEUA |
1264                                      DESC_HDR_MODE1_MDEU_INIT |
1265                                      DESC_HDR_MODE1_MDEU_PAD |
1266                                      DESC_HDR_MODE1_MDEU_SHA256_HMAC,
1267         },
1268         {
1269                 .name = "authenc(hmac(md5),cbc(aes))",
1270                 .driver_name = "authenc-hmac-md5-cbc-aes-talitos",
1271                 .blocksize = AES_BLOCK_SIZE,
1272                 .aead = {
1273                         .setkey = aead_authenc_setkey,
1274                         .setauthsize = aead_authenc_setauthsize,
1275                         .encrypt = aead_authenc_encrypt,
1276                         .decrypt = aead_authenc_decrypt,
1277                         .givencrypt = aead_authenc_givencrypt,
1278                         .geniv = "<built-in>",
1279                         .ivsize = AES_BLOCK_SIZE,
1280                         .maxauthsize = MD5_DIGEST_SIZE,
1281                         },
1282                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
1283                                      DESC_HDR_SEL0_AESU |
1284                                      DESC_HDR_MODE0_AESU_CBC |
1285                                      DESC_HDR_SEL1_MDEUA |
1286                                      DESC_HDR_MODE1_MDEU_INIT |
1287                                      DESC_HDR_MODE1_MDEU_PAD |
1288                                      DESC_HDR_MODE1_MDEU_MD5_HMAC,
1289         },
1290         {
1291                 .name = "authenc(hmac(md5),cbc(des3_ede))",
1292                 .driver_name = "authenc-hmac-md5-cbc-3des-talitos",
1293                 .blocksize = DES3_EDE_BLOCK_SIZE,
1294                 .aead = {
1295                         .setkey = aead_authenc_setkey,
1296                         .setauthsize = aead_authenc_setauthsize,
1297                         .encrypt = aead_authenc_encrypt,
1298                         .decrypt = aead_authenc_decrypt,
1299                         .givencrypt = aead_authenc_givencrypt,
1300                         .geniv = "<built-in>",
1301                         .ivsize = DES3_EDE_BLOCK_SIZE,
1302                         .maxauthsize = MD5_DIGEST_SIZE,
1303                         },
1304                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
1305                                      DESC_HDR_SEL0_DEU |
1306                                      DESC_HDR_MODE0_DEU_CBC |
1307                                      DESC_HDR_MODE0_DEU_3DES |
1308                                      DESC_HDR_SEL1_MDEUA |
1309                                      DESC_HDR_MODE1_MDEU_INIT |
1310                                      DESC_HDR_MODE1_MDEU_PAD |
1311                                      DESC_HDR_MODE1_MDEU_MD5_HMAC,
1312         }
1313 };
1314
1315 struct talitos_crypto_alg {
1316         struct list_head entry;
1317         struct device *dev;
1318         __be32 desc_hdr_template;
1319         struct crypto_alg crypto_alg;
1320 };
1321
1322 static int talitos_cra_init(struct crypto_tfm *tfm)
1323 {
1324         struct crypto_alg *alg = tfm->__crt_alg;
1325         struct talitos_crypto_alg *talitos_alg =
1326                  container_of(alg, struct talitos_crypto_alg, crypto_alg);
1327         struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
1328
1329         /* update context with ptr to dev */
1330         ctx->dev = talitos_alg->dev;
1331         /* copy descriptor header template value */
1332         ctx->desc_hdr_template = talitos_alg->desc_hdr_template;
1333
1334         /* random first IV */
1335         get_random_bytes(ctx->iv, TALITOS_MAX_IV_LENGTH);
1336
1337         return 0;
1338 }
1339
1340 /*
1341  * given the alg's descriptor header template, determine whether descriptor
1342  * type and primary/secondary execution units required match the hw
1343  * capabilities description provided in the device tree node.
1344  */
1345 static int hw_supports(struct device *dev, __be32 desc_hdr_template)
1346 {
1347         struct talitos_private *priv = dev_get_drvdata(dev);
1348         int ret;
1349
1350         ret = (1 << DESC_TYPE(desc_hdr_template) & priv->desc_types) &&
1351               (1 << PRIMARY_EU(desc_hdr_template) & priv->exec_units);
1352
1353         if (SECONDARY_EU(desc_hdr_template))
1354                 ret = ret && (1 << SECONDARY_EU(desc_hdr_template)
1355                               & priv->exec_units);
1356
1357         return ret;
1358 }
1359
1360 static int __devexit talitos_remove(struct of_device *ofdev)
1361 {
1362         struct device *dev = &ofdev->dev;
1363         struct talitos_private *priv = dev_get_drvdata(dev);
1364         struct talitos_crypto_alg *t_alg, *n;
1365         int i;
1366
1367         list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) {
1368                 crypto_unregister_alg(&t_alg->crypto_alg);
1369                 list_del(&t_alg->entry);
1370                 kfree(t_alg);
1371         }
1372
1373         if (hw_supports(dev, DESC_HDR_SEL0_RNG))
1374                 talitos_unregister_rng(dev);
1375
1376         kfree(priv->submit_count);
1377         kfree(priv->tail);
1378         kfree(priv->head);
1379
1380         if (priv->fifo)
1381                 for (i = 0; i < priv->num_channels; i++)
1382                         kfree(priv->fifo[i]);
1383
1384         kfree(priv->fifo);
1385         kfree(priv->head_lock);
1386         kfree(priv->tail_lock);
1387
1388         if (priv->irq != NO_IRQ) {
1389                 free_irq(priv->irq, dev);
1390                 irq_dispose_mapping(priv->irq);
1391         }
1392
1393         tasklet_kill(&priv->done_task);
1394         tasklet_kill(&priv->error_task);
1395
1396         iounmap(priv->reg);
1397
1398         dev_set_drvdata(dev, NULL);
1399
1400         kfree(priv);
1401
1402         return 0;
1403 }
1404
1405 static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev,
1406                                                     struct talitos_alg_template
1407                                                            *template)
1408 {
1409         struct talitos_crypto_alg *t_alg;
1410         struct crypto_alg *alg;
1411
1412         t_alg = kzalloc(sizeof(struct talitos_crypto_alg), GFP_KERNEL);
1413         if (!t_alg)
1414                 return ERR_PTR(-ENOMEM);
1415
1416         alg = &t_alg->crypto_alg;
1417
1418         snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", template->name);
1419         snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
1420                  template->driver_name);
1421         alg->cra_module = THIS_MODULE;
1422         alg->cra_init = talitos_cra_init;
1423         alg->cra_priority = TALITOS_CRA_PRIORITY;
1424         alg->cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC;
1425         alg->cra_blocksize = template->blocksize;
1426         alg->cra_alignmask = 0;
1427         alg->cra_type = &crypto_aead_type;
1428         alg->cra_ctxsize = sizeof(struct talitos_ctx);
1429         alg->cra_u.aead = template->aead;
1430
1431         t_alg->desc_hdr_template = template->desc_hdr_template;
1432         t_alg->dev = dev;
1433
1434         return t_alg;
1435 }
1436
1437 static int talitos_probe(struct of_device *ofdev,
1438                          const struct of_device_id *match)
1439 {
1440         struct device *dev = &ofdev->dev;
1441         struct device_node *np = ofdev->node;
1442         struct talitos_private *priv;
1443         const unsigned int *prop;
1444         int i, err;
1445
1446         priv = kzalloc(sizeof(struct talitos_private), GFP_KERNEL);
1447         if (!priv)
1448                 return -ENOMEM;
1449
1450         dev_set_drvdata(dev, priv);
1451
1452         priv->ofdev = ofdev;
1453
1454         INIT_LIST_HEAD(&priv->alg_list);
1455
1456         tasklet_init(&priv->done_task, talitos_done, (unsigned long)dev);
1457         tasklet_init(&priv->error_task, talitos_error, (unsigned long)dev);
1458
1459         priv->irq = irq_of_parse_and_map(np, 0);
1460
1461         if (priv->irq == NO_IRQ) {
1462                 dev_err(dev, "failed to map irq\n");
1463                 err = -EINVAL;
1464                 goto err_out;
1465         }
1466
1467         /* get the irq line */
1468         err = request_irq(priv->irq, talitos_interrupt, 0,
1469                           dev_driver_string(dev), dev);
1470         if (err) {
1471                 dev_err(dev, "failed to request irq %d\n", priv->irq);
1472                 irq_dispose_mapping(priv->irq);
1473                 priv->irq = NO_IRQ;
1474                 goto err_out;
1475         }
1476
1477         priv->reg = of_iomap(np, 0);
1478         if (!priv->reg) {
1479                 dev_err(dev, "failed to of_iomap\n");
1480                 err = -ENOMEM;
1481                 goto err_out;
1482         }
1483
1484         /* get SEC version capabilities from device tree */
1485         prop = of_get_property(np, "fsl,num-channels", NULL);
1486         if (prop)
1487                 priv->num_channels = *prop;
1488
1489         prop = of_get_property(np, "fsl,channel-fifo-len", NULL);
1490         if (prop)
1491                 priv->chfifo_len = *prop;
1492
1493         prop = of_get_property(np, "fsl,exec-units-mask", NULL);
1494         if (prop)
1495                 priv->exec_units = *prop;
1496
1497         prop = of_get_property(np, "fsl,descriptor-types-mask", NULL);
1498         if (prop)
1499                 priv->desc_types = *prop;
1500
1501         if (!is_power_of_2(priv->num_channels) || !priv->chfifo_len ||
1502             !priv->exec_units || !priv->desc_types) {
1503                 dev_err(dev, "invalid property data in device tree node\n");
1504                 err = -EINVAL;
1505                 goto err_out;
1506         }
1507
1508         if (of_device_is_compatible(np, "fsl,sec3.0"))
1509                 priv->features |= TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT;
1510
1511         priv->head_lock = kmalloc(sizeof(spinlock_t) * priv->num_channels,
1512                                   GFP_KERNEL);
1513         priv->tail_lock = kmalloc(sizeof(spinlock_t) * priv->num_channels,
1514                                   GFP_KERNEL);
1515         if (!priv->head_lock || !priv->tail_lock) {
1516                 dev_err(dev, "failed to allocate fifo locks\n");
1517                 err = -ENOMEM;
1518                 goto err_out;
1519         }
1520
1521         for (i = 0; i < priv->num_channels; i++) {
1522                 spin_lock_init(&priv->head_lock[i]);
1523                 spin_lock_init(&priv->tail_lock[i]);
1524         }
1525
1526         priv->fifo = kmalloc(sizeof(struct talitos_request *) *
1527                              priv->num_channels, GFP_KERNEL);
1528         if (!priv->fifo) {
1529                 dev_err(dev, "failed to allocate request fifo\n");
1530                 err = -ENOMEM;
1531                 goto err_out;
1532         }
1533
1534         priv->fifo_len = roundup_pow_of_two(priv->chfifo_len);
1535
1536         for (i = 0; i < priv->num_channels; i++) {
1537                 priv->fifo[i] = kzalloc(sizeof(struct talitos_request) *
1538                                         priv->fifo_len, GFP_KERNEL);
1539                 if (!priv->fifo[i]) {
1540                         dev_err(dev, "failed to allocate request fifo %d\n", i);
1541                         err = -ENOMEM;
1542                         goto err_out;
1543                 }
1544         }
1545
1546         priv->submit_count = kmalloc(sizeof(atomic_t) * priv->num_channels,
1547                                      GFP_KERNEL);
1548         if (!priv->submit_count) {
1549                 dev_err(dev, "failed to allocate fifo submit count space\n");
1550                 err = -ENOMEM;
1551                 goto err_out;
1552         }
1553         for (i = 0; i < priv->num_channels; i++)
1554                 atomic_set(&priv->submit_count[i], -priv->chfifo_len);
1555
1556         priv->head = kzalloc(sizeof(int) * priv->num_channels, GFP_KERNEL);
1557         priv->tail = kzalloc(sizeof(int) * priv->num_channels, GFP_KERNEL);
1558         if (!priv->head || !priv->tail) {
1559                 dev_err(dev, "failed to allocate request index space\n");
1560                 err = -ENOMEM;
1561                 goto err_out;
1562         }
1563
1564         /* reset and initialize the h/w */
1565         err = init_device(dev);
1566         if (err) {
1567                 dev_err(dev, "failed to initialize device\n");
1568                 goto err_out;
1569         }
1570
1571         /* register the RNG, if available */
1572         if (hw_supports(dev, DESC_HDR_SEL0_RNG)) {
1573                 err = talitos_register_rng(dev);
1574                 if (err) {
1575                         dev_err(dev, "failed to register hwrng: %d\n", err);
1576                         goto err_out;
1577                 } else
1578                         dev_info(dev, "hwrng\n");
1579         }
1580
1581         /* register crypto algorithms the device supports */
1582         for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
1583                 if (hw_supports(dev, driver_algs[i].desc_hdr_template)) {
1584                         struct talitos_crypto_alg *t_alg;
1585
1586                         t_alg = talitos_alg_alloc(dev, &driver_algs[i]);
1587                         if (IS_ERR(t_alg)) {
1588                                 err = PTR_ERR(t_alg);
1589                                 goto err_out;
1590                         }
1591
1592                         err = crypto_register_alg(&t_alg->crypto_alg);
1593                         if (err) {
1594                                 dev_err(dev, "%s alg registration failed\n",
1595                                         t_alg->crypto_alg.cra_driver_name);
1596                                 kfree(t_alg);
1597                         } else {
1598                                 list_add_tail(&t_alg->entry, &priv->alg_list);
1599                                 dev_info(dev, "%s\n",
1600                                          t_alg->crypto_alg.cra_driver_name);
1601                         }
1602                 }
1603         }
1604
1605         return 0;
1606
1607 err_out:
1608         talitos_remove(ofdev);
1609
1610         return err;
1611 }
1612
1613 static struct of_device_id talitos_match[] = {
1614         {
1615                 .compatible = "fsl,sec2.0",
1616         },
1617         {},
1618 };
1619 MODULE_DEVICE_TABLE(of, talitos_match);
1620
1621 static struct of_platform_driver talitos_driver = {
1622         .name = "talitos",
1623         .match_table = talitos_match,
1624         .probe = talitos_probe,
1625         .remove = __devexit_p(talitos_remove),
1626 };
1627
1628 static int __init talitos_init(void)
1629 {
1630         return of_register_platform_driver(&talitos_driver);
1631 }
1632 module_init(talitos_init);
1633
1634 static void __exit talitos_exit(void)
1635 {
1636         of_unregister_platform_driver(&talitos_driver);
1637 }
1638 module_exit(talitos_exit);
1639
1640 MODULE_LICENSE("GPL");
1641 MODULE_AUTHOR("Kim Phillips <kim.phillips@freescale.com>");
1642 MODULE_DESCRIPTION("Freescale integrated security engine (SEC) driver");