[CRYPTO] des: Create header file for common macros
[linux-3.10.git] / drivers / crypto / hifn_795x.c
1 /*
2  * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru>
3  * All rights reserved.
4  *
5  * This program is free software; you can redistribute it and/or modify
6  * it under the terms of the GNU General Public License as published by
7  * the Free Software Foundation; either version 2 of the License, or
8  * (at your option) any later version.
9  *
10  * This program is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13  * GNU General Public License for more details.
14  *
15  * You should have received a copy of the GNU General Public License
16  * along with this program; if not, write to the Free Software
17  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
18  */
19
20 #include <linux/kernel.h>
21 #include <linux/module.h>
22 #include <linux/mod_devicetable.h>
23 #include <linux/interrupt.h>
24 #include <linux/pci.h>
25 #include <linux/slab.h>
26 #include <linux/delay.h>
27 #include <linux/mm.h>
28 #include <linux/highmem.h>
29 #include <linux/crypto.h>
30
31 #include <crypto/algapi.h>
32
33 #include <asm/kmap_types.h>
34
35 #undef dprintk
36
37 #define HIFN_TEST
38 //#define HIFN_DEBUG
39
40 #ifdef HIFN_DEBUG
41 #define dprintk(f, a...)        printk(f, ##a)
42 #else
43 #define dprintk(f, a...)        do {} while (0)
44 #endif
45
46 static atomic_t hifn_dev_number;
47
48 #define ACRYPTO_OP_DECRYPT      0
49 #define ACRYPTO_OP_ENCRYPT      1
50 #define ACRYPTO_OP_HMAC         2
51 #define ACRYPTO_OP_RNG          3
52
53 #define ACRYPTO_MODE_ECB                0
54 #define ACRYPTO_MODE_CBC                1
55 #define ACRYPTO_MODE_CFB                2
56 #define ACRYPTO_MODE_OFB                3
57
58 #define ACRYPTO_TYPE_AES_128    0
59 #define ACRYPTO_TYPE_AES_192    1
60 #define ACRYPTO_TYPE_AES_256    2
61 #define ACRYPTO_TYPE_3DES       3
62 #define ACRYPTO_TYPE_DES        4
63
64 #define PCI_VENDOR_ID_HIFN              0x13A3
65 #define PCI_DEVICE_ID_HIFN_7955         0x0020
66 #define PCI_DEVICE_ID_HIFN_7956         0x001d
67
68 /* I/O region sizes */
69
70 #define HIFN_BAR0_SIZE                  0x1000
71 #define HIFN_BAR1_SIZE                  0x2000
72 #define HIFN_BAR2_SIZE                  0x8000
73
74 /* DMA registres */
75
76 #define HIFN_DMA_CRA                    0x0C    /* DMA Command Ring Address */
77 #define HIFN_DMA_SDRA                   0x1C    /* DMA Source Data Ring Address */
78 #define HIFN_DMA_RRA                    0x2C    /* DMA Result Ring Address */
79 #define HIFN_DMA_DDRA                   0x3C    /* DMA Destination Data Ring Address */
80 #define HIFN_DMA_STCTL                  0x40    /* DMA Status and Control */
81 #define HIFN_DMA_INTREN                 0x44    /* DMA Interrupt Enable */
82 #define HIFN_DMA_CFG1                   0x48    /* DMA Configuration #1 */
83 #define HIFN_DMA_CFG2                   0x6C    /* DMA Configuration #2 */
84 #define HIFN_CHIP_ID                    0x98    /* Chip ID */
85
86 /*
87  * Processing Unit Registers (offset from BASEREG0)
88  */
89 #define HIFN_0_PUDATA           0x00    /* Processing Unit Data */
90 #define HIFN_0_PUCTRL           0x04    /* Processing Unit Control */
91 #define HIFN_0_PUISR            0x08    /* Processing Unit Interrupt Status */
92 #define HIFN_0_PUCNFG           0x0c    /* Processing Unit Configuration */
93 #define HIFN_0_PUIER            0x10    /* Processing Unit Interrupt Enable */
94 #define HIFN_0_PUSTAT           0x14    /* Processing Unit Status/Chip ID */
95 #define HIFN_0_FIFOSTAT         0x18    /* FIFO Status */
96 #define HIFN_0_FIFOCNFG         0x1c    /* FIFO Configuration */
97 #define HIFN_0_SPACESIZE        0x20    /* Register space size */
98
99 /* Processing Unit Control Register (HIFN_0_PUCTRL) */
100 #define HIFN_PUCTRL_CLRSRCFIFO  0x0010  /* clear source fifo */
101 #define HIFN_PUCTRL_STOP        0x0008  /* stop pu */
102 #define HIFN_PUCTRL_LOCKRAM     0x0004  /* lock ram */
103 #define HIFN_PUCTRL_DMAENA      0x0002  /* enable dma */
104 #define HIFN_PUCTRL_RESET       0x0001  /* Reset processing unit */
105
106 /* Processing Unit Interrupt Status Register (HIFN_0_PUISR) */
107 #define HIFN_PUISR_CMDINVAL     0x8000  /* Invalid command interrupt */
108 #define HIFN_PUISR_DATAERR      0x4000  /* Data error interrupt */
109 #define HIFN_PUISR_SRCFIFO      0x2000  /* Source FIFO ready interrupt */
110 #define HIFN_PUISR_DSTFIFO      0x1000  /* Destination FIFO ready interrupt */
111 #define HIFN_PUISR_DSTOVER      0x0200  /* Destination overrun interrupt */
112 #define HIFN_PUISR_SRCCMD       0x0080  /* Source command interrupt */
113 #define HIFN_PUISR_SRCCTX       0x0040  /* Source context interrupt */
114 #define HIFN_PUISR_SRCDATA      0x0020  /* Source data interrupt */
115 #define HIFN_PUISR_DSTDATA      0x0010  /* Destination data interrupt */
116 #define HIFN_PUISR_DSTRESULT    0x0004  /* Destination result interrupt */
117
118 /* Processing Unit Configuration Register (HIFN_0_PUCNFG) */
119 #define HIFN_PUCNFG_DRAMMASK    0xe000  /* DRAM size mask */
120 #define HIFN_PUCNFG_DSZ_256K    0x0000  /* 256k dram */
121 #define HIFN_PUCNFG_DSZ_512K    0x2000  /* 512k dram */
122 #define HIFN_PUCNFG_DSZ_1M      0x4000  /* 1m dram */
123 #define HIFN_PUCNFG_DSZ_2M      0x6000  /* 2m dram */
124 #define HIFN_PUCNFG_DSZ_4M      0x8000  /* 4m dram */
125 #define HIFN_PUCNFG_DSZ_8M      0xa000  /* 8m dram */
126 #define HIFN_PUNCFG_DSZ_16M     0xc000  /* 16m dram */
127 #define HIFN_PUCNFG_DSZ_32M     0xe000  /* 32m dram */
128 #define HIFN_PUCNFG_DRAMREFRESH 0x1800  /* DRAM refresh rate mask */
129 #define HIFN_PUCNFG_DRFR_512    0x0000  /* 512 divisor of ECLK */
130 #define HIFN_PUCNFG_DRFR_256    0x0800  /* 256 divisor of ECLK */
131 #define HIFN_PUCNFG_DRFR_128    0x1000  /* 128 divisor of ECLK */
132 #define HIFN_PUCNFG_TCALLPHASES 0x0200  /* your guess is as good as mine... */
133 #define HIFN_PUCNFG_TCDRVTOTEM  0x0100  /* your guess is as good as mine... */
134 #define HIFN_PUCNFG_BIGENDIAN   0x0080  /* DMA big endian mode */
135 #define HIFN_PUCNFG_BUS32       0x0040  /* Bus width 32bits */
136 #define HIFN_PUCNFG_BUS16       0x0000  /* Bus width 16 bits */
137 #define HIFN_PUCNFG_CHIPID      0x0020  /* Allow chipid from PUSTAT */
138 #define HIFN_PUCNFG_DRAM        0x0010  /* Context RAM is DRAM */
139 #define HIFN_PUCNFG_SRAM        0x0000  /* Context RAM is SRAM */
140 #define HIFN_PUCNFG_COMPSING    0x0004  /* Enable single compression context */
141 #define HIFN_PUCNFG_ENCCNFG     0x0002  /* Encryption configuration */
142
143 /* Processing Unit Interrupt Enable Register (HIFN_0_PUIER) */
144 #define HIFN_PUIER_CMDINVAL     0x8000  /* Invalid command interrupt */
145 #define HIFN_PUIER_DATAERR      0x4000  /* Data error interrupt */
146 #define HIFN_PUIER_SRCFIFO      0x2000  /* Source FIFO ready interrupt */
147 #define HIFN_PUIER_DSTFIFO      0x1000  /* Destination FIFO ready interrupt */
148 #define HIFN_PUIER_DSTOVER      0x0200  /* Destination overrun interrupt */
149 #define HIFN_PUIER_SRCCMD       0x0080  /* Source command interrupt */
150 #define HIFN_PUIER_SRCCTX       0x0040  /* Source context interrupt */
151 #define HIFN_PUIER_SRCDATA      0x0020  /* Source data interrupt */
152 #define HIFN_PUIER_DSTDATA      0x0010  /* Destination data interrupt */
153 #define HIFN_PUIER_DSTRESULT    0x0004  /* Destination result interrupt */
154
155 /* Processing Unit Status Register/Chip ID (HIFN_0_PUSTAT) */
156 #define HIFN_PUSTAT_CMDINVAL    0x8000  /* Invalid command interrupt */
157 #define HIFN_PUSTAT_DATAERR     0x4000  /* Data error interrupt */
158 #define HIFN_PUSTAT_SRCFIFO     0x2000  /* Source FIFO ready interrupt */
159 #define HIFN_PUSTAT_DSTFIFO     0x1000  /* Destination FIFO ready interrupt */
160 #define HIFN_PUSTAT_DSTOVER     0x0200  /* Destination overrun interrupt */
161 #define HIFN_PUSTAT_SRCCMD      0x0080  /* Source command interrupt */
162 #define HIFN_PUSTAT_SRCCTX      0x0040  /* Source context interrupt */
163 #define HIFN_PUSTAT_SRCDATA     0x0020  /* Source data interrupt */
164 #define HIFN_PUSTAT_DSTDATA     0x0010  /* Destination data interrupt */
165 #define HIFN_PUSTAT_DSTRESULT   0x0004  /* Destination result interrupt */
166 #define HIFN_PUSTAT_CHIPREV     0x00ff  /* Chip revision mask */
167 #define HIFN_PUSTAT_CHIPENA     0xff00  /* Chip enabled mask */
168 #define HIFN_PUSTAT_ENA_2       0x1100  /* Level 2 enabled */
169 #define HIFN_PUSTAT_ENA_1       0x1000  /* Level 1 enabled */
170 #define HIFN_PUSTAT_ENA_0       0x3000  /* Level 0 enabled */
171 #define HIFN_PUSTAT_REV_2       0x0020  /* 7751 PT6/2 */
172 #define HIFN_PUSTAT_REV_3       0x0030  /* 7751 PT6/3 */
173
174 /* FIFO Status Register (HIFN_0_FIFOSTAT) */
175 #define HIFN_FIFOSTAT_SRC       0x7f00  /* Source FIFO available */
176 #define HIFN_FIFOSTAT_DST       0x007f  /* Destination FIFO available */
177
178 /* FIFO Configuration Register (HIFN_0_FIFOCNFG) */
179 #define HIFN_FIFOCNFG_THRESHOLD 0x0400  /* must be written as 1 */
180
181 /*
182  * DMA Interface Registers (offset from BASEREG1)
183  */
184 #define HIFN_1_DMA_CRAR         0x0c    /* DMA Command Ring Address */
185 #define HIFN_1_DMA_SRAR         0x1c    /* DMA Source Ring Address */
186 #define HIFN_1_DMA_RRAR         0x2c    /* DMA Result Ring Address */
187 #define HIFN_1_DMA_DRAR         0x3c    /* DMA Destination Ring Address */
188 #define HIFN_1_DMA_CSR          0x40    /* DMA Status and Control */
189 #define HIFN_1_DMA_IER          0x44    /* DMA Interrupt Enable */
190 #define HIFN_1_DMA_CNFG         0x48    /* DMA Configuration */
191 #define HIFN_1_PLL              0x4c    /* 795x: PLL config */
192 #define HIFN_1_7811_RNGENA      0x60    /* 7811: rng enable */
193 #define HIFN_1_7811_RNGCFG      0x64    /* 7811: rng config */
194 #define HIFN_1_7811_RNGDAT      0x68    /* 7811: rng data */
195 #define HIFN_1_7811_RNGSTS      0x6c    /* 7811: rng status */
196 #define HIFN_1_7811_MIPSRST     0x94    /* 7811: MIPS reset */
197 #define HIFN_1_REVID            0x98    /* Revision ID */
198 #define HIFN_1_UNLOCK_SECRET1   0xf4
199 #define HIFN_1_UNLOCK_SECRET2   0xfc
200 #define HIFN_1_PUB_RESET        0x204   /* Public/RNG Reset */
201 #define HIFN_1_PUB_BASE         0x300   /* Public Base Address */
202 #define HIFN_1_PUB_OPLEN        0x304   /* Public Operand Length */
203 #define HIFN_1_PUB_OP           0x308   /* Public Operand */
204 #define HIFN_1_PUB_STATUS       0x30c   /* Public Status */
205 #define HIFN_1_PUB_IEN          0x310   /* Public Interrupt enable */
206 #define HIFN_1_RNG_CONFIG       0x314   /* RNG config */
207 #define HIFN_1_RNG_DATA         0x318   /* RNG data */
208 #define HIFN_1_PUB_MEM          0x400   /* start of Public key memory */
209 #define HIFN_1_PUB_MEMEND       0xbff   /* end of Public key memory */
210
211 /* DMA Status and Control Register (HIFN_1_DMA_CSR) */
212 #define HIFN_DMACSR_D_CTRLMASK  0xc0000000      /* Destinition Ring Control */
213 #define HIFN_DMACSR_D_CTRL_NOP  0x00000000      /* Dest. Control: no-op */
214 #define HIFN_DMACSR_D_CTRL_DIS  0x40000000      /* Dest. Control: disable */
215 #define HIFN_DMACSR_D_CTRL_ENA  0x80000000      /* Dest. Control: enable */
216 #define HIFN_DMACSR_D_ABORT     0x20000000      /* Destinition Ring PCIAbort */
217 #define HIFN_DMACSR_D_DONE      0x10000000      /* Destinition Ring Done */
218 #define HIFN_DMACSR_D_LAST      0x08000000      /* Destinition Ring Last */
219 #define HIFN_DMACSR_D_WAIT      0x04000000      /* Destinition Ring Waiting */
220 #define HIFN_DMACSR_D_OVER      0x02000000      /* Destinition Ring Overflow */
221 #define HIFN_DMACSR_R_CTRL      0x00c00000      /* Result Ring Control */
222 #define HIFN_DMACSR_R_CTRL_NOP  0x00000000      /* Result Control: no-op */
223 #define HIFN_DMACSR_R_CTRL_DIS  0x00400000      /* Result Control: disable */
224 #define HIFN_DMACSR_R_CTRL_ENA  0x00800000      /* Result Control: enable */
225 #define HIFN_DMACSR_R_ABORT     0x00200000      /* Result Ring PCI Abort */
226 #define HIFN_DMACSR_R_DONE      0x00100000      /* Result Ring Done */
227 #define HIFN_DMACSR_R_LAST      0x00080000      /* Result Ring Last */
228 #define HIFN_DMACSR_R_WAIT      0x00040000      /* Result Ring Waiting */
229 #define HIFN_DMACSR_R_OVER      0x00020000      /* Result Ring Overflow */
230 #define HIFN_DMACSR_S_CTRL      0x0000c000      /* Source Ring Control */
231 #define HIFN_DMACSR_S_CTRL_NOP  0x00000000      /* Source Control: no-op */
232 #define HIFN_DMACSR_S_CTRL_DIS  0x00004000      /* Source Control: disable */
233 #define HIFN_DMACSR_S_CTRL_ENA  0x00008000      /* Source Control: enable */
234 #define HIFN_DMACSR_S_ABORT     0x00002000      /* Source Ring PCI Abort */
235 #define HIFN_DMACSR_S_DONE      0x00001000      /* Source Ring Done */
236 #define HIFN_DMACSR_S_LAST      0x00000800      /* Source Ring Last */
237 #define HIFN_DMACSR_S_WAIT      0x00000400      /* Source Ring Waiting */
238 #define HIFN_DMACSR_ILLW        0x00000200      /* Illegal write (7811 only) */
239 #define HIFN_DMACSR_ILLR        0x00000100      /* Illegal read (7811 only) */
240 #define HIFN_DMACSR_C_CTRL      0x000000c0      /* Command Ring Control */
241 #define HIFN_DMACSR_C_CTRL_NOP  0x00000000      /* Command Control: no-op */
242 #define HIFN_DMACSR_C_CTRL_DIS  0x00000040      /* Command Control: disable */
243 #define HIFN_DMACSR_C_CTRL_ENA  0x00000080      /* Command Control: enable */
244 #define HIFN_DMACSR_C_ABORT     0x00000020      /* Command Ring PCI Abort */
245 #define HIFN_DMACSR_C_DONE      0x00000010      /* Command Ring Done */
246 #define HIFN_DMACSR_C_LAST      0x00000008      /* Command Ring Last */
247 #define HIFN_DMACSR_C_WAIT      0x00000004      /* Command Ring Waiting */
248 #define HIFN_DMACSR_PUBDONE     0x00000002      /* Public op done (7951 only) */
249 #define HIFN_DMACSR_ENGINE      0x00000001      /* Command Ring Engine IRQ */
250
251 /* DMA Interrupt Enable Register (HIFN_1_DMA_IER) */
252 #define HIFN_DMAIER_D_ABORT     0x20000000      /* Destination Ring PCIAbort */
253 #define HIFN_DMAIER_D_DONE      0x10000000      /* Destination Ring Done */
254 #define HIFN_DMAIER_D_LAST      0x08000000      /* Destination Ring Last */
255 #define HIFN_DMAIER_D_WAIT      0x04000000      /* Destination Ring Waiting */
256 #define HIFN_DMAIER_D_OVER      0x02000000      /* Destination Ring Overflow */
257 #define HIFN_DMAIER_R_ABORT     0x00200000      /* Result Ring PCI Abort */
258 #define HIFN_DMAIER_R_DONE      0x00100000      /* Result Ring Done */
259 #define HIFN_DMAIER_R_LAST      0x00080000      /* Result Ring Last */
260 #define HIFN_DMAIER_R_WAIT      0x00040000      /* Result Ring Waiting */
261 #define HIFN_DMAIER_R_OVER      0x00020000      /* Result Ring Overflow */
262 #define HIFN_DMAIER_S_ABORT     0x00002000      /* Source Ring PCI Abort */
263 #define HIFN_DMAIER_S_DONE      0x00001000      /* Source Ring Done */
264 #define HIFN_DMAIER_S_LAST      0x00000800      /* Source Ring Last */
265 #define HIFN_DMAIER_S_WAIT      0x00000400      /* Source Ring Waiting */
266 #define HIFN_DMAIER_ILLW        0x00000200      /* Illegal write (7811 only) */
267 #define HIFN_DMAIER_ILLR        0x00000100      /* Illegal read (7811 only) */
268 #define HIFN_DMAIER_C_ABORT     0x00000020      /* Command Ring PCI Abort */
269 #define HIFN_DMAIER_C_DONE      0x00000010      /* Command Ring Done */
270 #define HIFN_DMAIER_C_LAST      0x00000008      /* Command Ring Last */
271 #define HIFN_DMAIER_C_WAIT      0x00000004      /* Command Ring Waiting */
272 #define HIFN_DMAIER_PUBDONE     0x00000002      /* public op done (7951 only) */
273 #define HIFN_DMAIER_ENGINE      0x00000001      /* Engine IRQ */
274
275 /* DMA Configuration Register (HIFN_1_DMA_CNFG) */
276 #define HIFN_DMACNFG_BIGENDIAN  0x10000000      /* big endian mode */
277 #define HIFN_DMACNFG_POLLFREQ   0x00ff0000      /* Poll frequency mask */
278 #define HIFN_DMACNFG_UNLOCK     0x00000800
279 #define HIFN_DMACNFG_POLLINVAL  0x00000700      /* Invalid Poll Scalar */
280 #define HIFN_DMACNFG_LAST       0x00000010      /* Host control LAST bit */
281 #define HIFN_DMACNFG_MODE       0x00000004      /* DMA mode */
282 #define HIFN_DMACNFG_DMARESET   0x00000002      /* DMA Reset # */
283 #define HIFN_DMACNFG_MSTRESET   0x00000001      /* Master Reset # */
284
285 #define HIFN_PLL_7956           0x00001d18      /* 7956 PLL config value */
286
287 /* Public key reset register (HIFN_1_PUB_RESET) */
288 #define HIFN_PUBRST_RESET       0x00000001      /* reset public/rng unit */
289
290 /* Public base address register (HIFN_1_PUB_BASE) */
291 #define HIFN_PUBBASE_ADDR       0x00003fff      /* base address */
292
293 /* Public operand length register (HIFN_1_PUB_OPLEN) */
294 #define HIFN_PUBOPLEN_MOD_M     0x0000007f      /* modulus length mask */
295 #define HIFN_PUBOPLEN_MOD_S     0               /* modulus length shift */
296 #define HIFN_PUBOPLEN_EXP_M     0x0003ff80      /* exponent length mask */
297 #define HIFN_PUBOPLEN_EXP_S     7               /* exponent lenght shift */
298 #define HIFN_PUBOPLEN_RED_M     0x003c0000      /* reducend length mask */
299 #define HIFN_PUBOPLEN_RED_S     18              /* reducend length shift */
300
301 /* Public operation register (HIFN_1_PUB_OP) */
302 #define HIFN_PUBOP_AOFFSET_M    0x0000007f      /* A offset mask */
303 #define HIFN_PUBOP_AOFFSET_S    0               /* A offset shift */
304 #define HIFN_PUBOP_BOFFSET_M    0x00000f80      /* B offset mask */
305 #define HIFN_PUBOP_BOFFSET_S    7               /* B offset shift */
306 #define HIFN_PUBOP_MOFFSET_M    0x0003f000      /* M offset mask */
307 #define HIFN_PUBOP_MOFFSET_S    12              /* M offset shift */
308 #define HIFN_PUBOP_OP_MASK      0x003c0000      /* Opcode: */
309 #define HIFN_PUBOP_OP_NOP       0x00000000      /*  NOP */
310 #define HIFN_PUBOP_OP_ADD       0x00040000      /*  ADD */
311 #define HIFN_PUBOP_OP_ADDC      0x00080000      /*  ADD w/carry */
312 #define HIFN_PUBOP_OP_SUB       0x000c0000      /*  SUB */
313 #define HIFN_PUBOP_OP_SUBC      0x00100000      /*  SUB w/carry */
314 #define HIFN_PUBOP_OP_MODADD    0x00140000      /*  Modular ADD */
315 #define HIFN_PUBOP_OP_MODSUB    0x00180000      /*  Modular SUB */
316 #define HIFN_PUBOP_OP_INCA      0x001c0000      /*  INC A */
317 #define HIFN_PUBOP_OP_DECA      0x00200000      /*  DEC A */
318 #define HIFN_PUBOP_OP_MULT      0x00240000      /*  MULT */
319 #define HIFN_PUBOP_OP_MODMULT   0x00280000      /*  Modular MULT */
320 #define HIFN_PUBOP_OP_MODRED    0x002c0000      /*  Modular RED */
321 #define HIFN_PUBOP_OP_MODEXP    0x00300000      /*  Modular EXP */
322
323 /* Public status register (HIFN_1_PUB_STATUS) */
324 #define HIFN_PUBSTS_DONE        0x00000001      /* operation done */
325 #define HIFN_PUBSTS_CARRY       0x00000002      /* carry */
326
327 /* Public interrupt enable register (HIFN_1_PUB_IEN) */
328 #define HIFN_PUBIEN_DONE        0x00000001      /* operation done interrupt */
329
330 /* Random number generator config register (HIFN_1_RNG_CONFIG) */
331 #define HIFN_RNGCFG_ENA         0x00000001      /* enable rng */
332
333 #define HIFN_NAMESIZE                   32
334 #define HIFN_MAX_RESULT_ORDER           5
335
336 #define HIFN_D_CMD_RSIZE                24*4
337 #define HIFN_D_SRC_RSIZE                80*4
338 #define HIFN_D_DST_RSIZE                80*4
339 #define HIFN_D_RES_RSIZE                24*4
340
341 #define HIFN_QUEUE_LENGTH               HIFN_D_CMD_RSIZE-5
342
343 #define AES_MIN_KEY_SIZE                16
344 #define AES_MAX_KEY_SIZE                32
345
346 #define HIFN_DES_KEY_LENGTH             8
347 #define HIFN_3DES_KEY_LENGTH            24
348 #define HIFN_MAX_CRYPT_KEY_LENGTH       AES_MAX_KEY_SIZE
349 #define HIFN_IV_LENGTH                  8
350 #define HIFN_AES_IV_LENGTH              16
351 #define HIFN_MAX_IV_LENGTH              HIFN_AES_IV_LENGTH
352
353 #define HIFN_MAC_KEY_LENGTH             64
354 #define HIFN_MD5_LENGTH                 16
355 #define HIFN_SHA1_LENGTH                20
356 #define HIFN_MAC_TRUNC_LENGTH           12
357
358 #define HIFN_MAX_COMMAND                (8 + 8 + 8 + 64 + 260)
359 #define HIFN_MAX_RESULT                 (8 + 4 + 4 + 20 + 4)
360 #define HIFN_USED_RESULT                12
361
362 struct hifn_desc
363 {
364         volatile u32            l;
365         volatile u32            p;
366 };
367
368 struct hifn_dma {
369         struct hifn_desc        cmdr[HIFN_D_CMD_RSIZE+1];
370         struct hifn_desc        srcr[HIFN_D_SRC_RSIZE+1];
371         struct hifn_desc        dstr[HIFN_D_DST_RSIZE+1];
372         struct hifn_desc        resr[HIFN_D_RES_RSIZE+1];
373
374         u8                      command_bufs[HIFN_D_CMD_RSIZE][HIFN_MAX_COMMAND];
375         u8                      result_bufs[HIFN_D_CMD_RSIZE][HIFN_MAX_RESULT];
376
377         u64                     test_src, test_dst;
378
379         /*
380          *  Our current positions for insertion and removal from the descriptor
381          *  rings.
382          */
383         volatile int            cmdi, srci, dsti, resi;
384         volatile int            cmdu, srcu, dstu, resu;
385         int                     cmdk, srck, dstk, resk;
386 };
387
388 #define HIFN_FLAG_CMD_BUSY      (1<<0)
389 #define HIFN_FLAG_SRC_BUSY      (1<<1)
390 #define HIFN_FLAG_DST_BUSY      (1<<2)
391 #define HIFN_FLAG_RES_BUSY      (1<<3)
392 #define HIFN_FLAG_OLD_KEY       (1<<4)
393
394 #define HIFN_DEFAULT_ACTIVE_NUM 5
395
396 struct hifn_device
397 {
398         char                    name[HIFN_NAMESIZE];
399
400         int                     irq;
401
402         struct pci_dev          *pdev;
403         void __iomem            *bar[3];
404
405         unsigned long           result_mem;
406         dma_addr_t              dst;
407
408         void                    *desc_virt;
409         dma_addr_t              desc_dma;
410
411         u32                     dmareg;
412
413         void                    *sa[HIFN_D_RES_RSIZE];
414
415         spinlock_t              lock;
416
417         void                    *priv;
418
419         u32                     flags;
420         int                     active, started;
421         struct delayed_work     work;
422         unsigned long           reset;
423         unsigned long           success;
424         unsigned long           prev_success;
425
426         u8                      snum;
427
428         struct crypto_queue     queue;
429         struct list_head        alg_list;
430 };
431
432 #define HIFN_D_LENGTH                   0x0000ffff
433 #define HIFN_D_NOINVALID                0x01000000
434 #define HIFN_D_MASKDONEIRQ              0x02000000
435 #define HIFN_D_DESTOVER                 0x04000000
436 #define HIFN_D_OVER                     0x08000000
437 #define HIFN_D_LAST                     0x20000000
438 #define HIFN_D_JUMP                     0x40000000
439 #define HIFN_D_VALID                    0x80000000
440
441 struct hifn_base_command
442 {
443         volatile u16            masks;
444         volatile u16            session_num;
445         volatile u16            total_source_count;
446         volatile u16            total_dest_count;
447 };
448
449 #define HIFN_BASE_CMD_COMP              0x0100  /* enable compression engine */
450 #define HIFN_BASE_CMD_PAD               0x0200  /* enable padding engine */
451 #define HIFN_BASE_CMD_MAC               0x0400  /* enable MAC engine */
452 #define HIFN_BASE_CMD_CRYPT             0x0800  /* enable crypt engine */
453 #define HIFN_BASE_CMD_DECODE            0x2000
454 #define HIFN_BASE_CMD_SRCLEN_M          0xc000
455 #define HIFN_BASE_CMD_SRCLEN_S          14
456 #define HIFN_BASE_CMD_DSTLEN_M          0x3000
457 #define HIFN_BASE_CMD_DSTLEN_S          12
458 #define HIFN_BASE_CMD_LENMASK_HI        0x30000
459 #define HIFN_BASE_CMD_LENMASK_LO        0x0ffff
460
461 /*
462  * Structure to help build up the command data structure.
463  */
464 struct hifn_crypt_command
465 {
466         volatile u16            masks;
467         volatile u16            header_skip;
468         volatile u16            source_count;
469         volatile u16            reserved;
470 };
471
472 #define HIFN_CRYPT_CMD_ALG_MASK         0x0003          /* algorithm: */
473 #define HIFN_CRYPT_CMD_ALG_DES          0x0000          /*   DES */
474 #define HIFN_CRYPT_CMD_ALG_3DES         0x0001          /*   3DES */
475 #define HIFN_CRYPT_CMD_ALG_RC4          0x0002          /*   RC4 */
476 #define HIFN_CRYPT_CMD_ALG_AES          0x0003          /*   AES */
477 #define HIFN_CRYPT_CMD_MODE_MASK        0x0018          /* Encrypt mode: */
478 #define HIFN_CRYPT_CMD_MODE_ECB         0x0000          /*   ECB */
479 #define HIFN_CRYPT_CMD_MODE_CBC         0x0008          /*   CBC */
480 #define HIFN_CRYPT_CMD_MODE_CFB         0x0010          /*   CFB */
481 #define HIFN_CRYPT_CMD_MODE_OFB         0x0018          /*   OFB */
482 #define HIFN_CRYPT_CMD_CLR_CTX          0x0040          /* clear context */
483 #define HIFN_CRYPT_CMD_KSZ_MASK         0x0600          /* AES key size: */
484 #define HIFN_CRYPT_CMD_KSZ_128          0x0000          /*  128 bit */
485 #define HIFN_CRYPT_CMD_KSZ_192          0x0200          /*  192 bit */
486 #define HIFN_CRYPT_CMD_KSZ_256          0x0400          /*  256 bit */
487 #define HIFN_CRYPT_CMD_NEW_KEY          0x0800          /* expect new key */
488 #define HIFN_CRYPT_CMD_NEW_IV           0x1000          /* expect new iv */
489 #define HIFN_CRYPT_CMD_SRCLEN_M         0xc000
490 #define HIFN_CRYPT_CMD_SRCLEN_S         14
491
492 /*
493  * Structure to help build up the command data structure.
494  */
495 struct hifn_mac_command
496 {
497         volatile u16            masks;
498         volatile u16            header_skip;
499         volatile u16            source_count;
500         volatile u16            reserved;
501 };
502
503 #define HIFN_MAC_CMD_ALG_MASK           0x0001
504 #define HIFN_MAC_CMD_ALG_SHA1           0x0000
505 #define HIFN_MAC_CMD_ALG_MD5            0x0001
506 #define HIFN_MAC_CMD_MODE_MASK          0x000c
507 #define HIFN_MAC_CMD_MODE_HMAC          0x0000
508 #define HIFN_MAC_CMD_MODE_SSL_MAC       0x0004
509 #define HIFN_MAC_CMD_MODE_HASH          0x0008
510 #define HIFN_MAC_CMD_MODE_FULL          0x0004
511 #define HIFN_MAC_CMD_TRUNC              0x0010
512 #define HIFN_MAC_CMD_RESULT             0x0020
513 #define HIFN_MAC_CMD_APPEND             0x0040
514 #define HIFN_MAC_CMD_SRCLEN_M           0xc000
515 #define HIFN_MAC_CMD_SRCLEN_S           14
516
517 /*
518  * MAC POS IPsec initiates authentication after encryption on encodes
519  * and before decryption on decodes.
520  */
521 #define HIFN_MAC_CMD_POS_IPSEC          0x0200
522 #define HIFN_MAC_CMD_NEW_KEY            0x0800
523
524 struct hifn_comp_command
525 {
526         volatile u16            masks;
527         volatile u16            header_skip;
528         volatile u16            source_count;
529         volatile u16            reserved;
530 };
531
532 #define HIFN_COMP_CMD_SRCLEN_M          0xc000
533 #define HIFN_COMP_CMD_SRCLEN_S          14
534 #define HIFN_COMP_CMD_ONE               0x0100  /* must be one */
535 #define HIFN_COMP_CMD_CLEARHIST         0x0010  /* clear history */
536 #define HIFN_COMP_CMD_UPDATEHIST        0x0008  /* update history */
537 #define HIFN_COMP_CMD_LZS_STRIP0        0x0004  /* LZS: strip zero */
538 #define HIFN_COMP_CMD_MPPC_RESTART      0x0004  /* MPPC: restart */
539 #define HIFN_COMP_CMD_ALG_MASK          0x0001  /* compression mode: */
540 #define HIFN_COMP_CMD_ALG_MPPC          0x0001  /*   MPPC */
541 #define HIFN_COMP_CMD_ALG_LZS           0x0000  /*   LZS */
542
543 struct hifn_base_result
544 {
545         volatile u16            flags;
546         volatile u16            session;
547         volatile u16            src_cnt;                /* 15:0 of source count */
548         volatile u16            dst_cnt;                /* 15:0 of dest count */
549 };
550
551 #define HIFN_BASE_RES_DSTOVERRUN        0x0200  /* destination overrun */
552 #define HIFN_BASE_RES_SRCLEN_M          0xc000  /* 17:16 of source count */
553 #define HIFN_BASE_RES_SRCLEN_S          14
554 #define HIFN_BASE_RES_DSTLEN_M          0x3000  /* 17:16 of dest count */
555 #define HIFN_BASE_RES_DSTLEN_S          12
556
557 struct hifn_comp_result
558 {
559         volatile u16            flags;
560         volatile u16            crc;
561 };
562
563 #define HIFN_COMP_RES_LCB_M             0xff00  /* longitudinal check byte */
564 #define HIFN_COMP_RES_LCB_S             8
565 #define HIFN_COMP_RES_RESTART           0x0004  /* MPPC: restart */
566 #define HIFN_COMP_RES_ENDMARKER         0x0002  /* LZS: end marker seen */
567 #define HIFN_COMP_RES_SRC_NOTZERO       0x0001  /* source expired */
568
569 struct hifn_mac_result
570 {
571         volatile u16            flags;
572         volatile u16            reserved;
573         /* followed by 0, 6, 8, or 10 u16's of the MAC, then crypt */
574 };
575
576 #define HIFN_MAC_RES_MISCOMPARE         0x0002  /* compare failed */
577 #define HIFN_MAC_RES_SRC_NOTZERO        0x0001  /* source expired */
578
579 struct hifn_crypt_result
580 {
581         volatile u16            flags;
582         volatile u16            reserved;
583 };
584
585 #define HIFN_CRYPT_RES_SRC_NOTZERO      0x0001  /* source expired */
586
587 #ifndef HIFN_POLL_FREQUENCY
588 #define HIFN_POLL_FREQUENCY     0x1
589 #endif
590
591 #ifndef HIFN_POLL_SCALAR
592 #define HIFN_POLL_SCALAR        0x0
593 #endif
594
595 #define HIFN_MAX_SEGLEN         0xffff          /* maximum dma segment len */
596 #define HIFN_MAX_DMALEN         0x3ffff         /* maximum dma length */
597
598 struct hifn_crypto_alg
599 {
600         struct list_head        entry;
601         struct crypto_alg       alg;
602         struct hifn_device      *dev;
603 };
604
605 #define ASYNC_SCATTERLIST_CACHE 16
606
607 #define ASYNC_FLAGS_MISALIGNED  (1<<0)
608
609 struct ablkcipher_walk
610 {
611         struct scatterlist      cache[ASYNC_SCATTERLIST_CACHE];
612         u32                     flags;
613         int                     num;
614 };
615
616 struct hifn_context
617 {
618         u8                      key[HIFN_MAX_CRYPT_KEY_LENGTH], *iv;
619         struct hifn_device      *dev;
620         unsigned int            keysize, ivsize;
621         u8                      op, type, mode, unused;
622         struct ablkcipher_walk  walk;
623         atomic_t                sg_num;
624 };
625
626 #define crypto_alg_to_hifn(alg) container_of(alg, struct hifn_crypto_alg, alg)
627
628 static inline u32 hifn_read_0(struct hifn_device *dev, u32 reg)
629 {
630         u32 ret;
631
632         ret = readl((char *)(dev->bar[0]) + reg);
633
634         return ret;
635 }
636
637 static inline u32 hifn_read_1(struct hifn_device *dev, u32 reg)
638 {
639         u32 ret;
640
641         ret = readl((char *)(dev->bar[1]) + reg);
642
643         return ret;
644 }
645
646 static inline void hifn_write_0(struct hifn_device *dev, u32 reg, u32 val)
647 {
648         writel(val, (char *)(dev->bar[0]) + reg);
649 }
650
651 static inline void hifn_write_1(struct hifn_device *dev, u32 reg, u32 val)
652 {
653         writel(val, (char *)(dev->bar[1]) + reg);
654 }
655
656 static void hifn_wait_puc(struct hifn_device *dev)
657 {
658         int i;
659         u32 ret;
660
661         for (i=10000; i > 0; --i) {
662                 ret = hifn_read_0(dev, HIFN_0_PUCTRL);
663                 if (!(ret & HIFN_PUCTRL_RESET))
664                         break;
665
666                 udelay(1);
667         }
668
669         if (!i)
670                 dprintk("%s: Failed to reset PUC unit.\n", dev->name);
671 }
672
673 static void hifn_reset_puc(struct hifn_device *dev)
674 {
675         hifn_write_0(dev, HIFN_0_PUCTRL, HIFN_PUCTRL_DMAENA);
676         hifn_wait_puc(dev);
677 }
678
679 static void hifn_stop_device(struct hifn_device *dev)
680 {
681         hifn_write_1(dev, HIFN_1_DMA_CSR,
682                 HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS |
683                 HIFN_DMACSR_S_CTRL_DIS | HIFN_DMACSR_C_CTRL_DIS);
684         hifn_write_0(dev, HIFN_0_PUIER, 0);
685         hifn_write_1(dev, HIFN_1_DMA_IER, 0);
686 }
687
688 static void hifn_reset_dma(struct hifn_device *dev, int full)
689 {
690         hifn_stop_device(dev);
691
692         /*
693          * Setting poll frequency and others to 0.
694          */
695         hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
696                         HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
697         mdelay(1);
698
699         /*
700          * Reset DMA.
701          */
702         if (full) {
703                 hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MODE);
704                 mdelay(1);
705         } else {
706                 hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MODE |
707                                 HIFN_DMACNFG_MSTRESET);
708                 hifn_reset_puc(dev);
709         }
710
711         hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
712                         HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
713
714         hifn_reset_puc(dev);
715 }
716
717 static u32 hifn_next_signature(u_int32_t a, u_int cnt)
718 {
719         int i;
720         u32 v;
721
722         for (i = 0; i < cnt; i++) {
723
724                 /* get the parity */
725                 v = a & 0x80080125;
726                 v ^= v >> 16;
727                 v ^= v >> 8;
728                 v ^= v >> 4;
729                 v ^= v >> 2;
730                 v ^= v >> 1;
731
732                 a = (v & 1) ^ (a << 1);
733         }
734
735         return a;
736 }
737
738 static struct pci2id {
739         u_short         pci_vendor;
740         u_short         pci_prod;
741         char            card_id[13];
742 } pci2id[] = {
743         {
744                 PCI_VENDOR_ID_HIFN,
745                 PCI_DEVICE_ID_HIFN_7955,
746                 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
747                   0x00, 0x00, 0x00, 0x00, 0x00 }
748         },
749         {
750                 PCI_VENDOR_ID_HIFN,
751                 PCI_DEVICE_ID_HIFN_7956,
752                 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
753                   0x00, 0x00, 0x00, 0x00, 0x00 }
754         }
755 };
756
757 static int hifn_init_pubrng(struct hifn_device *dev)
758 {
759         int i;
760
761         hifn_write_1(dev, HIFN_1_PUB_RESET, hifn_read_1(dev, HIFN_1_PUB_RESET) |
762                         HIFN_PUBRST_RESET);
763
764         for (i=100; i > 0; --i) {
765                 mdelay(1);
766
767                 if ((hifn_read_1(dev, HIFN_1_PUB_RESET) & HIFN_PUBRST_RESET) == 0)
768                         break;
769         }
770
771         if (!i)
772                 dprintk("Chip %s: Failed to initialise public key engine.\n",
773                                 dev->name);
774         else {
775                 hifn_write_1(dev, HIFN_1_PUB_IEN, HIFN_PUBIEN_DONE);
776                 dev->dmareg |= HIFN_DMAIER_PUBDONE;
777                 hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg);
778
779                 dprintk("Chip %s: Public key engine has been sucessfully "
780                                 "initialised.\n", dev->name);
781         }
782
783         /*
784          * Enable RNG engine.
785          */
786
787         hifn_write_1(dev, HIFN_1_RNG_CONFIG,
788                         hifn_read_1(dev, HIFN_1_RNG_CONFIG) | HIFN_RNGCFG_ENA);
789         dprintk("Chip %s: RNG engine has been successfully initialised.\n",
790                         dev->name);
791
792         return 0;
793 }
794
795 static int hifn_enable_crypto(struct hifn_device *dev)
796 {
797         u32 dmacfg, addr;
798         char *offtbl = NULL;
799         int i;
800
801         for (i = 0; i < sizeof(pci2id)/sizeof(pci2id[0]); i++) {
802                 if (pci2id[i].pci_vendor == dev->pdev->vendor &&
803                                 pci2id[i].pci_prod == dev->pdev->device) {
804                         offtbl = pci2id[i].card_id;
805                         break;
806                 }
807         }
808
809         if (offtbl == NULL) {
810                 dprintk("Chip %s: Unknown card!\n", dev->name);
811                 return -ENODEV;
812         }
813
814         dmacfg = hifn_read_1(dev, HIFN_1_DMA_CNFG);
815
816         hifn_write_1(dev, HIFN_1_DMA_CNFG,
817                         HIFN_DMACNFG_UNLOCK | HIFN_DMACNFG_MSTRESET |
818                         HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
819         mdelay(1);
820         addr = hifn_read_1(dev, HIFN_1_UNLOCK_SECRET1);
821         mdelay(1);
822         hifn_write_1(dev, HIFN_1_UNLOCK_SECRET2, 0);
823         mdelay(1);
824
825         for (i=0; i<12; ++i) {
826                 addr = hifn_next_signature(addr, offtbl[i] + 0x101);
827                 hifn_write_1(dev, HIFN_1_UNLOCK_SECRET2, addr);
828
829                 mdelay(1);
830         }
831         hifn_write_1(dev, HIFN_1_DMA_CNFG, dmacfg);
832
833         dprintk("Chip %s: %s.\n", dev->name, pci_name(dev->pdev));
834
835         return 0;
836 }
837
838 static void hifn_init_dma(struct hifn_device *dev)
839 {
840         struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
841         u32 dptr = dev->desc_dma;
842         int i;
843
844         for (i=0; i<HIFN_D_CMD_RSIZE; ++i)
845                 dma->cmdr[i].p = __cpu_to_le32(dptr +
846                                 offsetof(struct hifn_dma, command_bufs[i][0]));
847         for (i=0; i<HIFN_D_RES_RSIZE; ++i)
848                 dma->resr[i].p = __cpu_to_le32(dptr +
849                                 offsetof(struct hifn_dma, result_bufs[i][0]));
850
851         /*
852          * Setup LAST descriptors.
853          */
854         dma->cmdr[HIFN_D_CMD_RSIZE].p = __cpu_to_le32(dptr +
855                         offsetof(struct hifn_dma, cmdr[0]));
856         dma->srcr[HIFN_D_SRC_RSIZE].p = __cpu_to_le32(dptr +
857                         offsetof(struct hifn_dma, srcr[0]));
858         dma->dstr[HIFN_D_DST_RSIZE].p = __cpu_to_le32(dptr +
859                         offsetof(struct hifn_dma, dstr[0]));
860         dma->resr[HIFN_D_RES_RSIZE].p = __cpu_to_le32(dptr +
861                         offsetof(struct hifn_dma, resr[0]));
862
863         dma->cmdu = dma->srcu = dma->dstu = dma->resu = 0;
864         dma->cmdi = dma->srci = dma->dsti = dma->resi = 0;
865         dma->cmdk = dma->srck = dma->dstk = dma->resk = 0;
866 }
867
868 static void hifn_init_registers(struct hifn_device *dev)
869 {
870         u32 dptr = dev->desc_dma;
871
872         /* Initialization magic... */
873         hifn_write_0(dev, HIFN_0_PUCTRL, HIFN_PUCTRL_DMAENA);
874         hifn_write_0(dev, HIFN_0_FIFOCNFG, HIFN_FIFOCNFG_THRESHOLD);
875         hifn_write_0(dev, HIFN_0_PUIER, HIFN_PUIER_DSTOVER);
876
877         /* write all 4 ring address registers */
878         hifn_write_1(dev, HIFN_1_DMA_CRAR, __cpu_to_le32(dptr +
879                                 offsetof(struct hifn_dma, cmdr[0])));
880         hifn_write_1(dev, HIFN_1_DMA_SRAR, __cpu_to_le32(dptr +
881                                 offsetof(struct hifn_dma, srcr[0])));
882         hifn_write_1(dev, HIFN_1_DMA_DRAR, __cpu_to_le32(dptr +
883                                 offsetof(struct hifn_dma, dstr[0])));
884         hifn_write_1(dev, HIFN_1_DMA_RRAR, __cpu_to_le32(dptr +
885                                 offsetof(struct hifn_dma, resr[0])));
886
887         mdelay(2);
888 #if 0
889         hifn_write_1(dev, HIFN_1_DMA_CSR,
890             HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS |
891             HIFN_DMACSR_S_CTRL_DIS | HIFN_DMACSR_C_CTRL_DIS |
892             HIFN_DMACSR_D_ABORT | HIFN_DMACSR_D_DONE | HIFN_DMACSR_D_LAST |
893             HIFN_DMACSR_D_WAIT | HIFN_DMACSR_D_OVER |
894             HIFN_DMACSR_R_ABORT | HIFN_DMACSR_R_DONE | HIFN_DMACSR_R_LAST |
895             HIFN_DMACSR_R_WAIT | HIFN_DMACSR_R_OVER |
896             HIFN_DMACSR_S_ABORT | HIFN_DMACSR_S_DONE | HIFN_DMACSR_S_LAST |
897             HIFN_DMACSR_S_WAIT |
898             HIFN_DMACSR_C_ABORT | HIFN_DMACSR_C_DONE | HIFN_DMACSR_C_LAST |
899             HIFN_DMACSR_C_WAIT |
900             HIFN_DMACSR_ENGINE |
901             HIFN_DMACSR_PUBDONE);
902 #else
903         hifn_write_1(dev, HIFN_1_DMA_CSR,
904             HIFN_DMACSR_C_CTRL_ENA | HIFN_DMACSR_S_CTRL_ENA |
905             HIFN_DMACSR_D_CTRL_ENA | HIFN_DMACSR_R_CTRL_ENA |
906             HIFN_DMACSR_D_ABORT | HIFN_DMACSR_D_DONE | HIFN_DMACSR_D_LAST |
907             HIFN_DMACSR_D_WAIT | HIFN_DMACSR_D_OVER |
908             HIFN_DMACSR_R_ABORT | HIFN_DMACSR_R_DONE | HIFN_DMACSR_R_LAST |
909             HIFN_DMACSR_R_WAIT | HIFN_DMACSR_R_OVER |
910             HIFN_DMACSR_S_ABORT | HIFN_DMACSR_S_DONE | HIFN_DMACSR_S_LAST |
911             HIFN_DMACSR_S_WAIT |
912             HIFN_DMACSR_C_ABORT | HIFN_DMACSR_C_DONE | HIFN_DMACSR_C_LAST |
913             HIFN_DMACSR_C_WAIT |
914             HIFN_DMACSR_ENGINE |
915             HIFN_DMACSR_PUBDONE);
916 #endif
917         hifn_read_1(dev, HIFN_1_DMA_CSR);
918
919         dev->dmareg |= HIFN_DMAIER_R_DONE | HIFN_DMAIER_C_ABORT |
920             HIFN_DMAIER_D_OVER | HIFN_DMAIER_R_OVER |
921             HIFN_DMAIER_S_ABORT | HIFN_DMAIER_D_ABORT | HIFN_DMAIER_R_ABORT |
922             HIFN_DMAIER_ENGINE;
923         dev->dmareg &= ~HIFN_DMAIER_C_WAIT;
924
925         hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg);
926         hifn_read_1(dev, HIFN_1_DMA_IER);
927 #if 0
928         hifn_write_0(dev, HIFN_0_PUCNFG, HIFN_PUCNFG_ENCCNFG |
929                     HIFN_PUCNFG_DRFR_128 | HIFN_PUCNFG_TCALLPHASES |
930                     HIFN_PUCNFG_TCDRVTOTEM | HIFN_PUCNFG_BUS32 |
931                     HIFN_PUCNFG_DRAM);
932 #else
933         hifn_write_0(dev, HIFN_0_PUCNFG, 0x10342);
934 #endif
935         hifn_write_1(dev, HIFN_1_PLL, HIFN_PLL_7956);
936
937         hifn_write_0(dev, HIFN_0_PUISR, HIFN_PUISR_DSTOVER);
938         hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
939             HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE | HIFN_DMACNFG_LAST |
940             ((HIFN_POLL_FREQUENCY << 16 ) & HIFN_DMACNFG_POLLFREQ) |
941             ((HIFN_POLL_SCALAR << 8) & HIFN_DMACNFG_POLLINVAL));
942 }
943
944 static int hifn_setup_base_command(struct hifn_device *dev, u8 *buf,
945                 unsigned dlen, unsigned slen, u16 mask, u8 snum)
946 {
947         struct hifn_base_command *base_cmd;
948         u8 *buf_pos = buf;
949
950         base_cmd = (struct hifn_base_command *)buf_pos;
951         base_cmd->masks = __cpu_to_le16(mask);
952         base_cmd->total_source_count =
953                 __cpu_to_le16(slen & HIFN_BASE_CMD_LENMASK_LO);
954         base_cmd->total_dest_count =
955                 __cpu_to_le16(dlen & HIFN_BASE_CMD_LENMASK_LO);
956
957         dlen >>= 16;
958         slen >>= 16;
959         base_cmd->session_num = __cpu_to_le16(snum |
960             ((slen << HIFN_BASE_CMD_SRCLEN_S) & HIFN_BASE_CMD_SRCLEN_M) |
961             ((dlen << HIFN_BASE_CMD_DSTLEN_S) & HIFN_BASE_CMD_DSTLEN_M));
962
963         return sizeof(struct hifn_base_command);
964 }
965
966 static int hifn_setup_crypto_command(struct hifn_device *dev,
967                 u8 *buf, unsigned dlen, unsigned slen,
968                 u8 *key, int keylen, u8 *iv, int ivsize, u16 mode)
969 {
970         struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
971         struct hifn_crypt_command *cry_cmd;
972         u8 *buf_pos = buf;
973         u16 cmd_len;
974
975         cry_cmd = (struct hifn_crypt_command *)buf_pos;
976
977         cry_cmd->source_count = __cpu_to_le16(dlen & 0xffff);
978         dlen >>= 16;
979         cry_cmd->masks = __cpu_to_le16(mode |
980                         ((dlen << HIFN_CRYPT_CMD_SRCLEN_S) &
981                          HIFN_CRYPT_CMD_SRCLEN_M));
982         cry_cmd->header_skip = 0;
983         cry_cmd->reserved = 0;
984
985         buf_pos += sizeof(struct hifn_crypt_command);
986
987         dma->cmdu++;
988         if (dma->cmdu > 1) {
989                 dev->dmareg |= HIFN_DMAIER_C_WAIT;
990                 hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg);
991         }
992
993         if (keylen) {
994                 memcpy(buf_pos, key, keylen);
995                 buf_pos += keylen;
996         }
997         if (ivsize) {
998                 memcpy(buf_pos, iv, ivsize);
999                 buf_pos += ivsize;
1000         }
1001
1002         cmd_len = buf_pos - buf;
1003
1004         return cmd_len;
1005 }
1006
1007 static int hifn_setup_src_desc(struct hifn_device *dev, struct page *page,
1008                 unsigned int offset, unsigned int size)
1009 {
1010         struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
1011         int idx;
1012         dma_addr_t addr;
1013
1014         addr = pci_map_page(dev->pdev, page, offset, size, PCI_DMA_TODEVICE);
1015
1016         idx = dma->srci;
1017
1018         dma->srcr[idx].p = __cpu_to_le32(addr);
1019         dma->srcr[idx].l = __cpu_to_le32(size) | HIFN_D_VALID |
1020                         HIFN_D_MASKDONEIRQ | HIFN_D_NOINVALID | HIFN_D_LAST;
1021
1022         if (++idx == HIFN_D_SRC_RSIZE) {
1023                 dma->srcr[idx].l = __cpu_to_le32(HIFN_D_VALID |
1024                                 HIFN_D_JUMP |
1025                                 HIFN_D_MASKDONEIRQ | HIFN_D_LAST);
1026                 idx = 0;
1027         }
1028
1029         dma->srci = idx;
1030         dma->srcu++;
1031
1032         if (!(dev->flags & HIFN_FLAG_SRC_BUSY)) {
1033                 hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_S_CTRL_ENA);
1034                 dev->flags |= HIFN_FLAG_SRC_BUSY;
1035         }
1036
1037         return size;
1038 }
1039
1040 static void hifn_setup_res_desc(struct hifn_device *dev)
1041 {
1042         struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
1043
1044         dma->resr[dma->resi].l = __cpu_to_le32(HIFN_USED_RESULT |
1045                         HIFN_D_VALID | HIFN_D_LAST);
1046         /*
1047          * dma->resr[dma->resi].l = __cpu_to_le32(HIFN_MAX_RESULT | HIFN_D_VALID |
1048          *                                      HIFN_D_LAST | HIFN_D_NOINVALID);
1049          */
1050
1051         if (++dma->resi == HIFN_D_RES_RSIZE) {
1052                 dma->resr[HIFN_D_RES_RSIZE].l = __cpu_to_le32(HIFN_D_VALID |
1053                                 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ | HIFN_D_LAST);
1054                 dma->resi = 0;
1055         }
1056
1057         dma->resu++;
1058
1059         if (!(dev->flags & HIFN_FLAG_RES_BUSY)) {
1060                 hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_R_CTRL_ENA);
1061                 dev->flags |= HIFN_FLAG_RES_BUSY;
1062         }
1063 }
1064
1065 static void hifn_setup_dst_desc(struct hifn_device *dev, struct page *page,
1066                 unsigned offset, unsigned size)
1067 {
1068         struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
1069         int idx;
1070         dma_addr_t addr;
1071
1072         addr = pci_map_page(dev->pdev, page, offset, size, PCI_DMA_FROMDEVICE);
1073
1074         idx = dma->dsti;
1075         dma->dstr[idx].p = __cpu_to_le32(addr);
1076         dma->dstr[idx].l = __cpu_to_le32(size | HIFN_D_VALID |
1077                         HIFN_D_MASKDONEIRQ | HIFN_D_NOINVALID | HIFN_D_LAST);
1078
1079         if (++idx == HIFN_D_DST_RSIZE) {
1080                 dma->dstr[idx].l = __cpu_to_le32(HIFN_D_VALID |
1081                                 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ |
1082                                 HIFN_D_LAST | HIFN_D_NOINVALID);
1083                 idx = 0;
1084         }
1085         dma->dsti = idx;
1086         dma->dstu++;
1087
1088         if (!(dev->flags & HIFN_FLAG_DST_BUSY)) {
1089                 hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_D_CTRL_ENA);
1090                 dev->flags |= HIFN_FLAG_DST_BUSY;
1091         }
1092 }
1093
1094 static int hifn_setup_dma(struct hifn_device *dev, struct page *spage, unsigned int soff,
1095                 struct page *dpage, unsigned int doff, unsigned int nbytes, void *priv,
1096                 struct hifn_context *ctx)
1097 {
1098         struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
1099         int cmd_len, sa_idx;
1100         u8 *buf, *buf_pos;
1101         u16 mask;
1102
1103         dprintk("%s: spage: %p, soffset: %u, dpage: %p, doffset: %u, nbytes: %u, priv: %p, ctx: %p.\n",
1104                         dev->name, spage, soff, dpage, doff, nbytes, priv, ctx);
1105
1106         sa_idx = dma->resi;
1107
1108         hifn_setup_src_desc(dev, spage, soff, nbytes);
1109
1110         buf_pos = buf = dma->command_bufs[dma->cmdi];
1111
1112         mask = 0;
1113         switch (ctx->op) {
1114                 case ACRYPTO_OP_DECRYPT:
1115                         mask = HIFN_BASE_CMD_CRYPT | HIFN_BASE_CMD_DECODE;
1116                         break;
1117                 case ACRYPTO_OP_ENCRYPT:
1118                         mask = HIFN_BASE_CMD_CRYPT;
1119                         break;
1120                 case ACRYPTO_OP_HMAC:
1121                         mask = HIFN_BASE_CMD_MAC;
1122                         break;
1123                 default:
1124                         goto err_out;
1125         }
1126
1127         buf_pos += hifn_setup_base_command(dev, buf_pos, nbytes,
1128                         nbytes, mask, dev->snum);
1129
1130         if (ctx->op == ACRYPTO_OP_ENCRYPT || ctx->op == ACRYPTO_OP_DECRYPT) {
1131                 u16 md = 0;
1132
1133                 if (ctx->keysize)
1134                         md |= HIFN_CRYPT_CMD_NEW_KEY;
1135                 if (ctx->iv && ctx->mode != ACRYPTO_MODE_ECB)
1136                         md |= HIFN_CRYPT_CMD_NEW_IV;
1137
1138                 switch (ctx->mode) {
1139                         case ACRYPTO_MODE_ECB:
1140                                 md |= HIFN_CRYPT_CMD_MODE_ECB;
1141                                 break;
1142                         case ACRYPTO_MODE_CBC:
1143                                 md |= HIFN_CRYPT_CMD_MODE_CBC;
1144                                 break;
1145                         case ACRYPTO_MODE_CFB:
1146                                 md |= HIFN_CRYPT_CMD_MODE_CFB;
1147                                 break;
1148                         case ACRYPTO_MODE_OFB:
1149                                 md |= HIFN_CRYPT_CMD_MODE_OFB;
1150                                 break;
1151                         default:
1152                                 goto err_out;
1153                 }
1154
1155                 switch (ctx->type) {
1156                         case ACRYPTO_TYPE_AES_128:
1157                                 if (ctx->keysize != 16)
1158                                         goto err_out;
1159                                 md |= HIFN_CRYPT_CMD_KSZ_128 |
1160                                         HIFN_CRYPT_CMD_ALG_AES;
1161                                 break;
1162                         case ACRYPTO_TYPE_AES_192:
1163                                 if (ctx->keysize != 24)
1164                                         goto err_out;
1165                                 md |= HIFN_CRYPT_CMD_KSZ_192 |
1166                                         HIFN_CRYPT_CMD_ALG_AES;
1167                                 break;
1168                         case ACRYPTO_TYPE_AES_256:
1169                                 if (ctx->keysize != 32)
1170                                         goto err_out;
1171                                 md |= HIFN_CRYPT_CMD_KSZ_256 |
1172                                         HIFN_CRYPT_CMD_ALG_AES;
1173                                 break;
1174                         case ACRYPTO_TYPE_3DES:
1175                                 if (ctx->keysize != 24)
1176                                         goto err_out;
1177                                 md |= HIFN_CRYPT_CMD_ALG_3DES;
1178                                 break;
1179                         case ACRYPTO_TYPE_DES:
1180                                 if (ctx->keysize != 8)
1181                                         goto err_out;
1182                                 md |= HIFN_CRYPT_CMD_ALG_DES;
1183                                 break;
1184                         default:
1185                                 goto err_out;
1186                 }
1187
1188                 buf_pos += hifn_setup_crypto_command(dev, buf_pos,
1189                                 nbytes, nbytes, ctx->key, ctx->keysize,
1190                                 ctx->iv, ctx->ivsize, md);
1191         }
1192
1193         dev->sa[sa_idx] = priv;
1194
1195         cmd_len = buf_pos - buf;
1196         dma->cmdr[dma->cmdi].l = __cpu_to_le32(cmd_len | HIFN_D_VALID |
1197                         HIFN_D_LAST | HIFN_D_MASKDONEIRQ);
1198
1199         if (++dma->cmdi == HIFN_D_CMD_RSIZE) {
1200                 dma->cmdr[dma->cmdi].l = __cpu_to_le32(HIFN_MAX_COMMAND |
1201                         HIFN_D_VALID | HIFN_D_LAST |
1202                         HIFN_D_MASKDONEIRQ | HIFN_D_JUMP);
1203                 dma->cmdi = 0;
1204         } else
1205                 dma->cmdr[dma->cmdi-1].l |= __cpu_to_le32(HIFN_D_VALID);
1206
1207         if (!(dev->flags & HIFN_FLAG_CMD_BUSY)) {
1208                 hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_C_CTRL_ENA);
1209                 dev->flags |= HIFN_FLAG_CMD_BUSY;
1210         }
1211
1212         hifn_setup_dst_desc(dev, dpage, doff, nbytes);
1213         hifn_setup_res_desc(dev);
1214
1215         return 0;
1216
1217 err_out:
1218         return -EINVAL;
1219 }
1220
1221 static int ablkcipher_walk_init(struct ablkcipher_walk *w,
1222                 int num, gfp_t gfp_flags)
1223 {
1224         int i;
1225
1226         num = min(ASYNC_SCATTERLIST_CACHE, num);
1227         sg_init_table(w->cache, num);
1228
1229         w->num = 0;
1230         for (i=0; i<num; ++i) {
1231                 struct page *page = alloc_page(gfp_flags);
1232                 struct scatterlist *s;
1233
1234                 if (!page)
1235                         break;
1236
1237                 s = &w->cache[i];
1238
1239                 sg_set_page(s, page, PAGE_SIZE, 0);
1240                 w->num++;
1241         }
1242
1243         return i;
1244 }
1245
1246 static void ablkcipher_walk_exit(struct ablkcipher_walk *w)
1247 {
1248         int i;
1249
1250         for (i=0; i<w->num; ++i) {
1251                 struct scatterlist *s = &w->cache[i];
1252
1253                 __free_page(sg_page(s));
1254
1255                 s->length = 0;
1256         }
1257
1258         w->num = 0;
1259 }
1260
1261 static int ablkcipher_add(void *daddr, unsigned int *drestp, struct scatterlist *src,
1262                 unsigned int size, unsigned int *nbytesp)
1263 {
1264         unsigned int copy, drest = *drestp, nbytes = *nbytesp;
1265         int idx = 0;
1266         void *saddr;
1267
1268         if (drest < size || size > nbytes)
1269                 return -EINVAL;
1270
1271         while (size) {
1272                 copy = min(drest, src->length);
1273
1274                 saddr = kmap_atomic(sg_page(src), KM_SOFTIRQ1);
1275                 memcpy(daddr, saddr + src->offset, copy);
1276                 kunmap_atomic(saddr, KM_SOFTIRQ1);
1277
1278                 size -= copy;
1279                 drest -= copy;
1280                 nbytes -= copy;
1281                 daddr += copy;
1282
1283                 dprintk("%s: copy: %u, size: %u, drest: %u, nbytes: %u.\n",
1284                                 __func__, copy, size, drest, nbytes);
1285
1286                 src++;
1287                 idx++;
1288         }
1289
1290         *nbytesp = nbytes;
1291         *drestp = drest;
1292
1293         return idx;
1294 }
1295
1296 static int ablkcipher_walk(struct ablkcipher_request *req,
1297                 struct ablkcipher_walk *w)
1298 {
1299         unsigned blocksize =
1300                 crypto_ablkcipher_blocksize(crypto_ablkcipher_reqtfm(req));
1301         unsigned alignmask =
1302                 crypto_ablkcipher_alignmask(crypto_ablkcipher_reqtfm(req));
1303         struct scatterlist *src, *dst, *t;
1304         void *daddr;
1305         unsigned int nbytes = req->nbytes, offset, copy, diff;
1306         int idx, tidx, err;
1307
1308         tidx = idx = 0;
1309         offset = 0;
1310         while (nbytes) {
1311                 if (idx >= w->num && (w->flags & ASYNC_FLAGS_MISALIGNED))
1312                         return -EINVAL;
1313
1314                 src = &req->src[idx];
1315                 dst = &req->dst[idx];
1316
1317                 dprintk("\n%s: slen: %u, dlen: %u, soff: %u, doff: %u, offset: %u, "
1318                                 "blocksize: %u, nbytes: %u.\n",
1319                                 __func__, src->length, dst->length, src->offset,
1320                                 dst->offset, offset, blocksize, nbytes);
1321
1322                 if (src->length & (blocksize - 1) ||
1323                                 src->offset & (alignmask - 1) ||
1324                                 dst->length & (blocksize - 1) ||
1325                                 dst->offset & (alignmask - 1) ||
1326                                 offset) {
1327                         unsigned slen = src->length - offset;
1328                         unsigned dlen = PAGE_SIZE;
1329
1330                         t = &w->cache[idx];
1331
1332                         daddr = kmap_atomic(sg_page(t), KM_SOFTIRQ0);
1333                         err = ablkcipher_add(daddr, &dlen, src, slen, &nbytes);
1334                         if (err < 0)
1335                                 goto err_out_unmap;
1336
1337                         idx += err;
1338
1339                         copy = slen & ~(blocksize - 1);
1340                         diff = slen & (blocksize - 1);
1341
1342                         if (dlen < nbytes) {
1343                                 /*
1344                                  * Destination page does not have enough space
1345                                  * to put there additional blocksized chunk,
1346                                  * so we mark that page as containing only
1347                                  * blocksize aligned chunks:
1348                                  *      t->length = (slen & ~(blocksize - 1));
1349                                  * and increase number of bytes to be processed
1350                                  * in next chunk:
1351                                  *      nbytes += diff;
1352                                  */
1353                                 nbytes += diff;
1354
1355                                 /*
1356                                  * Temporary of course...
1357                                  * Kick author if you will catch this one.
1358                                  */
1359                                 printk(KERN_ERR "%s: dlen: %u, nbytes: %u,"
1360                                         "slen: %u, offset: %u.\n",
1361                                         __func__, dlen, nbytes, slen, offset);
1362                                 printk(KERN_ERR "%s: please contact author to fix this "
1363                                         "issue, generally you should not catch "
1364                                         "this path under any condition but who "
1365                                         "knows how did you use crypto code.\n"
1366                                         "Thank you.\n", __func__);
1367                                 BUG();
1368                         } else {
1369                                 copy += diff + nbytes;
1370
1371                                 src = &req->src[idx];
1372
1373                                 err = ablkcipher_add(daddr + slen, &dlen, src, nbytes, &nbytes);
1374                                 if (err < 0)
1375                                         goto err_out_unmap;
1376
1377                                 idx += err;
1378                         }
1379
1380                         t->length = copy;
1381                         t->offset = offset;
1382
1383                         kunmap_atomic(daddr, KM_SOFTIRQ0);
1384                 } else {
1385                         nbytes -= src->length;
1386                         idx++;
1387                 }
1388
1389                 tidx++;
1390         }
1391
1392         return tidx;
1393
1394 err_out_unmap:
1395         kunmap_atomic(daddr, KM_SOFTIRQ0);
1396         return err;
1397 }
1398
1399 static int hifn_setup_session(struct ablkcipher_request *req)
1400 {
1401         struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm);
1402         struct hifn_device *dev = ctx->dev;
1403         struct page *spage, *dpage;
1404         unsigned long soff, doff, flags;
1405         unsigned int nbytes = req->nbytes, idx = 0, len;
1406         int err = -EINVAL, sg_num;
1407         struct scatterlist *src, *dst, *t;
1408         unsigned blocksize =
1409                 crypto_ablkcipher_blocksize(crypto_ablkcipher_reqtfm(req));
1410         unsigned alignmask =
1411                 crypto_ablkcipher_alignmask(crypto_ablkcipher_reqtfm(req));
1412
1413         if (ctx->iv && !ctx->ivsize && ctx->mode != ACRYPTO_MODE_ECB)
1414                 goto err_out_exit;
1415
1416         ctx->walk.flags = 0;
1417
1418         while (nbytes) {
1419                 src = &req->src[idx];
1420                 dst = &req->dst[idx];
1421
1422                 if (src->length & (blocksize - 1) ||
1423                                 src->offset & (alignmask - 1) ||
1424                                 dst->length & (blocksize - 1) ||
1425                                 dst->offset & (alignmask - 1)) {
1426                         ctx->walk.flags |= ASYNC_FLAGS_MISALIGNED;
1427                 }
1428
1429                 nbytes -= src->length;
1430                 idx++;
1431         }
1432
1433         if (ctx->walk.flags & ASYNC_FLAGS_MISALIGNED) {
1434                 err = ablkcipher_walk_init(&ctx->walk, idx, GFP_ATOMIC);
1435                 if (err < 0)
1436                         return err;
1437         }
1438
1439         nbytes = req->nbytes;
1440         idx = 0;
1441
1442         sg_num = ablkcipher_walk(req, &ctx->walk);
1443
1444         atomic_set(&ctx->sg_num, sg_num);
1445
1446         spin_lock_irqsave(&dev->lock, flags);
1447         if (dev->started + sg_num > HIFN_QUEUE_LENGTH) {
1448                 err = -EAGAIN;
1449                 goto err_out;
1450         }
1451
1452         dev->snum++;
1453         dev->started += sg_num;
1454
1455         while (nbytes) {
1456                 src = &req->src[idx];
1457                 dst = &req->dst[idx];
1458                 t = &ctx->walk.cache[idx];
1459
1460                 if (t->length) {
1461                         spage = dpage = sg_page(t);
1462                         soff = doff = 0;
1463                         len = t->length;
1464                 } else {
1465                         spage = sg_page(src);
1466                         soff = src->offset;
1467
1468                         dpage = sg_page(dst);
1469                         doff = dst->offset;
1470
1471                         len = dst->length;
1472                 }
1473
1474                 idx++;
1475
1476                 err = hifn_setup_dma(dev, spage, soff, dpage, doff, nbytes,
1477                                 req, ctx);
1478                 if (err)
1479                         goto err_out;
1480
1481                 nbytes -= len;
1482         }
1483
1484         dev->active = HIFN_DEFAULT_ACTIVE_NUM;
1485         spin_unlock_irqrestore(&dev->lock, flags);
1486
1487         return 0;
1488
1489 err_out:
1490         spin_unlock_irqrestore(&dev->lock, flags);
1491 err_out_exit:
1492         if (err && printk_ratelimit())
1493                 dprintk("%s: iv: %p [%d], key: %p [%d], mode: %u, op: %u, "
1494                                 "type: %u, err: %d.\n",
1495                         dev->name, ctx->iv, ctx->ivsize,
1496                         ctx->key, ctx->keysize,
1497                         ctx->mode, ctx->op, ctx->type, err);
1498
1499         return err;
1500 }
1501
1502 static int hifn_test(struct hifn_device *dev, int encdec, u8 snum)
1503 {
1504         int n, err;
1505         u8 src[16];
1506         struct hifn_context ctx;
1507         u8 fips_aes_ecb_from_zero[16] = {
1508                 0x66, 0xE9, 0x4B, 0xD4,
1509                 0xEF, 0x8A, 0x2C, 0x3B,
1510                 0x88, 0x4C, 0xFA, 0x59,
1511                 0xCA, 0x34, 0x2B, 0x2E};
1512
1513         memset(src, 0, sizeof(src));
1514         memset(ctx.key, 0, sizeof(ctx.key));
1515
1516         ctx.dev = dev;
1517         ctx.keysize = 16;
1518         ctx.ivsize = 0;
1519         ctx.iv = NULL;
1520         ctx.op = (encdec)?ACRYPTO_OP_ENCRYPT:ACRYPTO_OP_DECRYPT;
1521         ctx.mode = ACRYPTO_MODE_ECB;
1522         ctx.type = ACRYPTO_TYPE_AES_128;
1523         atomic_set(&ctx.sg_num, 1);
1524
1525         err = hifn_setup_dma(dev,
1526                         virt_to_page(src), offset_in_page(src),
1527                         virt_to_page(src), offset_in_page(src),
1528                         sizeof(src), NULL, &ctx);
1529         if (err)
1530                 goto err_out;
1531
1532         msleep(200);
1533
1534         dprintk("%s: decoded: ", dev->name);
1535         for (n=0; n<sizeof(src); ++n)
1536                 dprintk("%02x ", src[n]);
1537         dprintk("\n");
1538         dprintk("%s: FIPS   : ", dev->name);
1539         for (n=0; n<sizeof(fips_aes_ecb_from_zero); ++n)
1540                 dprintk("%02x ", fips_aes_ecb_from_zero[n]);
1541         dprintk("\n");
1542
1543         if (!memcmp(src, fips_aes_ecb_from_zero, sizeof(fips_aes_ecb_from_zero))) {
1544                 printk(KERN_INFO "%s: AES 128 ECB test has been successfully "
1545                                 "passed.\n", dev->name);
1546                 return 0;
1547         }
1548
1549 err_out:
1550         printk(KERN_INFO "%s: AES 128 ECB test has been failed.\n", dev->name);
1551         return -1;
1552 }
1553
1554 static int hifn_start_device(struct hifn_device *dev)
1555 {
1556         int err;
1557
1558         hifn_reset_dma(dev, 1);
1559
1560         err = hifn_enable_crypto(dev);
1561         if (err)
1562                 return err;
1563
1564         hifn_reset_puc(dev);
1565
1566         hifn_init_dma(dev);
1567
1568         hifn_init_registers(dev);
1569
1570         hifn_init_pubrng(dev);
1571
1572         return 0;
1573 }
1574
1575 static int ablkcipher_get(void *saddr, unsigned int *srestp, unsigned int offset,
1576                 struct scatterlist *dst, unsigned int size, unsigned int *nbytesp)
1577 {
1578         unsigned int srest = *srestp, nbytes = *nbytesp, copy;
1579         void *daddr;
1580         int idx = 0;
1581
1582         if (srest < size || size > nbytes)
1583                 return -EINVAL;
1584
1585         while (size) {
1586
1587                 copy = min(dst->length, srest);
1588
1589                 daddr = kmap_atomic(sg_page(dst), KM_IRQ0);
1590                 memcpy(daddr + dst->offset + offset, saddr, copy);
1591                 kunmap_atomic(daddr, KM_IRQ0);
1592
1593                 nbytes -= copy;
1594                 size -= copy;
1595                 srest -= copy;
1596                 saddr += copy;
1597                 offset = 0;
1598
1599                 dprintk("%s: copy: %u, size: %u, srest: %u, nbytes: %u.\n",
1600                                 __func__, copy, size, srest, nbytes);
1601
1602                 dst++;
1603                 idx++;
1604         }
1605
1606         *nbytesp = nbytes;
1607         *srestp = srest;
1608
1609         return idx;
1610 }
1611
1612 static void hifn_process_ready(struct ablkcipher_request *req, int error)
1613 {
1614         struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm);
1615         struct hifn_device *dev;
1616
1617         dprintk("%s: req: %p, ctx: %p.\n", __func__, req, ctx);
1618
1619         dev = ctx->dev;
1620         dprintk("%s: req: %p, started: %d, sg_num: %d.\n",
1621                 __func__, req, dev->started, atomic_read(&ctx->sg_num));
1622
1623         if (--dev->started < 0)
1624                 BUG();
1625
1626         if (atomic_dec_and_test(&ctx->sg_num)) {
1627                 unsigned int nbytes = req->nbytes;
1628                 int idx = 0, err;
1629                 struct scatterlist *dst, *t;
1630                 void *saddr;
1631
1632                 if (ctx->walk.flags & ASYNC_FLAGS_MISALIGNED) {
1633                         while (nbytes) {
1634                                 t = &ctx->walk.cache[idx];
1635                                 dst = &req->dst[idx];
1636
1637                                 dprintk("\n%s: sg_page(t): %p, t->length: %u, "
1638                                         "sg_page(dst): %p, dst->length: %u, "
1639                                         "nbytes: %u.\n",
1640                                         __func__, sg_page(t), t->length,
1641                                         sg_page(dst), dst->length, nbytes);
1642
1643                                 if (!t->length) {
1644                                         nbytes -= dst->length;
1645                                         idx++;
1646                                         continue;
1647                                 }
1648
1649                                 saddr = kmap_atomic(sg_page(t), KM_IRQ1);
1650
1651                                 err = ablkcipher_get(saddr, &t->length, t->offset,
1652                                                 dst, nbytes, &nbytes);
1653                                 if (err < 0) {
1654                                         kunmap_atomic(saddr, KM_IRQ1);
1655                                         break;
1656                                 }
1657
1658                                 idx += err;
1659                                 kunmap_atomic(saddr, KM_IRQ1);
1660                         }
1661
1662                         ablkcipher_walk_exit(&ctx->walk);
1663                 }
1664
1665                 req->base.complete(&req->base, error);
1666         }
1667 }
1668
1669 static void hifn_check_for_completion(struct hifn_device *dev, int error)
1670 {
1671         int i;
1672         struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
1673
1674         for (i=0; i<HIFN_D_RES_RSIZE; ++i) {
1675                 struct hifn_desc *d = &dma->resr[i];
1676
1677                 if (!(d->l & __cpu_to_le32(HIFN_D_VALID)) && dev->sa[i]) {
1678                         dev->success++;
1679                         dev->reset = 0;
1680                         hifn_process_ready(dev->sa[i], error);
1681                         dev->sa[i] = NULL;
1682                 }
1683
1684                 if (d->l & __cpu_to_le32(HIFN_D_DESTOVER | HIFN_D_OVER))
1685                         if (printk_ratelimit())
1686                                 printk("%s: overflow detected [d: %u, o: %u] "
1687                                                 "at %d resr: l: %08x, p: %08x.\n",
1688                                         dev->name,
1689                                         !!(d->l & __cpu_to_le32(HIFN_D_DESTOVER)),
1690                                         !!(d->l & __cpu_to_le32(HIFN_D_OVER)),
1691                                         i, d->l, d->p);
1692         }
1693 }
1694
1695 static void hifn_clear_rings(struct hifn_device *dev)
1696 {
1697         struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
1698         int i, u;
1699
1700         dprintk("%s: ring cleanup 1: i: %d.%d.%d.%d, u: %d.%d.%d.%d, "
1701                         "k: %d.%d.%d.%d.\n",
1702                         dev->name,
1703                         dma->cmdi, dma->srci, dma->dsti, dma->resi,
1704                         dma->cmdu, dma->srcu, dma->dstu, dma->resu,
1705                         dma->cmdk, dma->srck, dma->dstk, dma->resk);
1706
1707         i = dma->resk; u = dma->resu;
1708         while (u != 0) {
1709                 if (dma->resr[i].l & __cpu_to_le32(HIFN_D_VALID))
1710                         break;
1711
1712                 if (i != HIFN_D_RES_RSIZE)
1713                         u--;
1714
1715                 if (++i == (HIFN_D_RES_RSIZE + 1))
1716                         i = 0;
1717         }
1718         dma->resk = i; dma->resu = u;
1719
1720         i = dma->srck; u = dma->srcu;
1721         while (u != 0) {
1722                 if (i == HIFN_D_SRC_RSIZE)
1723                         i = 0;
1724                 if (dma->srcr[i].l & __cpu_to_le32(HIFN_D_VALID))
1725                         break;
1726                 i++, u--;
1727         }
1728         dma->srck = i; dma->srcu = u;
1729
1730         i = dma->cmdk; u = dma->cmdu;
1731         while (u != 0) {
1732                 if (dma->cmdr[i].l & __cpu_to_le32(HIFN_D_VALID))
1733                         break;
1734                 if (i != HIFN_D_CMD_RSIZE)
1735                         u--;
1736                 if (++i == (HIFN_D_CMD_RSIZE + 1))
1737                         i = 0;
1738         }
1739         dma->cmdk = i; dma->cmdu = u;
1740
1741         i = dma->dstk; u = dma->dstu;
1742         while (u != 0) {
1743                 if (i == HIFN_D_DST_RSIZE)
1744                         i = 0;
1745                 if (dma->dstr[i].l & __cpu_to_le32(HIFN_D_VALID))
1746                         break;
1747                 i++, u--;
1748         }
1749         dma->dstk = i; dma->dstu = u;
1750
1751         dprintk("%s: ring cleanup 2: i: %d.%d.%d.%d, u: %d.%d.%d.%d, "
1752                         "k: %d.%d.%d.%d.\n",
1753                         dev->name,
1754                         dma->cmdi, dma->srci, dma->dsti, dma->resi,
1755                         dma->cmdu, dma->srcu, dma->dstu, dma->resu,
1756                         dma->cmdk, dma->srck, dma->dstk, dma->resk);
1757 }
1758
1759 static void hifn_work(struct work_struct *work)
1760 {
1761         struct delayed_work *dw = container_of(work, struct delayed_work, work);
1762         struct hifn_device *dev = container_of(dw, struct hifn_device, work);
1763         unsigned long flags;
1764         int reset = 0;
1765         u32 r = 0;
1766
1767         spin_lock_irqsave(&dev->lock, flags);
1768         if (dev->active == 0) {
1769                 struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
1770
1771                 if (dma->cmdu == 0 && (dev->flags & HIFN_FLAG_CMD_BUSY)) {
1772                         dev->flags &= ~HIFN_FLAG_CMD_BUSY;
1773                         r |= HIFN_DMACSR_C_CTRL_DIS;
1774                 }
1775                 if (dma->srcu == 0 && (dev->flags & HIFN_FLAG_SRC_BUSY)) {
1776                         dev->flags &= ~HIFN_FLAG_SRC_BUSY;
1777                         r |= HIFN_DMACSR_S_CTRL_DIS;
1778                 }
1779                 if (dma->dstu == 0 && (dev->flags & HIFN_FLAG_DST_BUSY)) {
1780                         dev->flags &= ~HIFN_FLAG_DST_BUSY;
1781                         r |= HIFN_DMACSR_D_CTRL_DIS;
1782                 }
1783                 if (dma->resu == 0 && (dev->flags & HIFN_FLAG_RES_BUSY)) {
1784                         dev->flags &= ~HIFN_FLAG_RES_BUSY;
1785                         r |= HIFN_DMACSR_R_CTRL_DIS;
1786                 }
1787                 if (r)
1788                         hifn_write_1(dev, HIFN_1_DMA_CSR, r);
1789         } else
1790                 dev->active--;
1791
1792         if (dev->prev_success == dev->success && dev->started)
1793                 reset = 1;
1794         dev->prev_success = dev->success;
1795         spin_unlock_irqrestore(&dev->lock, flags);
1796
1797         if (reset) {
1798                 dprintk("%s: r: %08x, active: %d, started: %d, "
1799                                 "success: %lu: reset: %d.\n",
1800                         dev->name, r, dev->active, dev->started,
1801                         dev->success, reset);
1802
1803                 if (++dev->reset >= 5) {
1804                         dprintk("%s: really hard reset.\n", dev->name);
1805                         hifn_reset_dma(dev, 1);
1806                         hifn_stop_device(dev);
1807                         hifn_start_device(dev);
1808                         dev->reset = 0;
1809                 }
1810
1811                 spin_lock_irqsave(&dev->lock, flags);
1812                 hifn_check_for_completion(dev, -EBUSY);
1813                 hifn_clear_rings(dev);
1814                 dev->started = 0;
1815                 spin_unlock_irqrestore(&dev->lock, flags);
1816         }
1817
1818         schedule_delayed_work(&dev->work, HZ);
1819 }
1820
1821 static irqreturn_t hifn_interrupt(int irq, void *data)
1822 {
1823         struct hifn_device *dev = (struct hifn_device *)data;
1824         struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
1825         u32 dmacsr, restart;
1826
1827         dmacsr = hifn_read_1(dev, HIFN_1_DMA_CSR);
1828
1829         dprintk("%s: 1 dmacsr: %08x, dmareg: %08x, res: %08x [%d], "
1830                         "i: %d.%d.%d.%d, u: %d.%d.%d.%d.\n",
1831                 dev->name, dmacsr, dev->dmareg, dmacsr & dev->dmareg, dma->cmdi,
1832                 dma->cmdu, dma->srcu, dma->dstu, dma->resu,
1833                 dma->cmdi, dma->srci, dma->dsti, dma->resi);
1834
1835         if ((dmacsr & dev->dmareg) == 0)
1836                 return IRQ_NONE;
1837
1838         hifn_write_1(dev, HIFN_1_DMA_CSR, dmacsr & dev->dmareg);
1839
1840         if (dmacsr & HIFN_DMACSR_ENGINE)
1841                 hifn_write_0(dev, HIFN_0_PUISR, hifn_read_0(dev, HIFN_0_PUISR));
1842         if (dmacsr & HIFN_DMACSR_PUBDONE)
1843                 hifn_write_1(dev, HIFN_1_PUB_STATUS,
1844                         hifn_read_1(dev, HIFN_1_PUB_STATUS) | HIFN_PUBSTS_DONE);
1845
1846         restart = dmacsr & (HIFN_DMACSR_R_OVER | HIFN_DMACSR_D_OVER);
1847         if (restart) {
1848                 u32 puisr = hifn_read_0(dev, HIFN_0_PUISR);
1849
1850                 if (printk_ratelimit())
1851                         printk("%s: overflow: r: %d, d: %d, puisr: %08x, d: %u.\n",
1852                                 dev->name, !!(dmacsr & HIFN_DMACSR_R_OVER),
1853                                 !!(dmacsr & HIFN_DMACSR_D_OVER),
1854                                 puisr, !!(puisr & HIFN_PUISR_DSTOVER));
1855                 if (!!(puisr & HIFN_PUISR_DSTOVER))
1856                         hifn_write_0(dev, HIFN_0_PUISR, HIFN_PUISR_DSTOVER);
1857                 hifn_write_1(dev, HIFN_1_DMA_CSR, dmacsr & (HIFN_DMACSR_R_OVER |
1858                                         HIFN_DMACSR_D_OVER));
1859         }
1860
1861         restart = dmacsr & (HIFN_DMACSR_C_ABORT | HIFN_DMACSR_S_ABORT |
1862                         HIFN_DMACSR_D_ABORT | HIFN_DMACSR_R_ABORT);
1863         if (restart) {
1864                 if (printk_ratelimit())
1865                         printk("%s: abort: c: %d, s: %d, d: %d, r: %d.\n",
1866                                 dev->name, !!(dmacsr & HIFN_DMACSR_C_ABORT),
1867                                 !!(dmacsr & HIFN_DMACSR_S_ABORT),
1868                                 !!(dmacsr & HIFN_DMACSR_D_ABORT),
1869                                 !!(dmacsr & HIFN_DMACSR_R_ABORT));
1870                 hifn_reset_dma(dev, 1);
1871                 hifn_init_dma(dev);
1872                 hifn_init_registers(dev);
1873         }
1874
1875         if ((dmacsr & HIFN_DMACSR_C_WAIT) && (dma->cmdu == 0)) {
1876                 dprintk("%s: wait on command.\n", dev->name);
1877                 dev->dmareg &= ~(HIFN_DMAIER_C_WAIT);
1878                 hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg);
1879         }
1880
1881         hifn_check_for_completion(dev, 0);
1882         hifn_clear_rings(dev);
1883
1884         return IRQ_HANDLED;
1885 }
1886
1887 static void hifn_flush(struct hifn_device *dev)
1888 {
1889         unsigned long flags;
1890         struct crypto_async_request *async_req;
1891         struct hifn_context *ctx;
1892         struct ablkcipher_request *req;
1893         struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
1894         int i;
1895
1896         spin_lock_irqsave(&dev->lock, flags);
1897         for (i=0; i<HIFN_D_RES_RSIZE; ++i) {
1898                 struct hifn_desc *d = &dma->resr[i];
1899
1900                 if (dev->sa[i]) {
1901                         hifn_process_ready(dev->sa[i],
1902                                 (d->l & __cpu_to_le32(HIFN_D_VALID))?-ENODEV:0);
1903                 }
1904         }
1905
1906         while ((async_req = crypto_dequeue_request(&dev->queue))) {
1907                 ctx = crypto_tfm_ctx(async_req->tfm);
1908                 req = container_of(async_req, struct ablkcipher_request, base);
1909
1910                 hifn_process_ready(req, -ENODEV);
1911         }
1912         spin_unlock_irqrestore(&dev->lock, flags);
1913 }
1914
1915 static int hifn_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
1916                 unsigned int len)
1917 {
1918         struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
1919         struct hifn_context *ctx = crypto_tfm_ctx(tfm);
1920         struct hifn_device *dev = ctx->dev;
1921
1922         if (len > HIFN_MAX_CRYPT_KEY_LENGTH) {
1923                 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
1924                 return -1;
1925         }
1926
1927         dev->flags &= ~HIFN_FLAG_OLD_KEY;
1928
1929         memcpy(ctx->key, key, len);
1930         ctx->keysize = len;
1931
1932         return 0;
1933 }
1934
1935 static int hifn_handle_req(struct ablkcipher_request *req)
1936 {
1937         struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm);
1938         struct hifn_device *dev = ctx->dev;
1939         int err = -EAGAIN;
1940
1941         if (dev->started + DIV_ROUND_UP(req->nbytes, PAGE_SIZE) <= HIFN_QUEUE_LENGTH)
1942                 err = hifn_setup_session(req);
1943
1944         if (err == -EAGAIN) {
1945                 unsigned long flags;
1946
1947                 spin_lock_irqsave(&dev->lock, flags);
1948                 err = ablkcipher_enqueue_request(&dev->queue, req);
1949                 spin_unlock_irqrestore(&dev->lock, flags);
1950         }
1951
1952         return err;
1953 }
1954
1955 static int hifn_setup_crypto_req(struct ablkcipher_request *req, u8 op,
1956                 u8 type, u8 mode)
1957 {
1958         struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm);
1959         unsigned ivsize;
1960
1961         ivsize = crypto_ablkcipher_ivsize(crypto_ablkcipher_reqtfm(req));
1962
1963         if (req->info && mode != ACRYPTO_MODE_ECB) {
1964                 if (type == ACRYPTO_TYPE_AES_128)
1965                         ivsize = HIFN_AES_IV_LENGTH;
1966                 else if (type == ACRYPTO_TYPE_DES)
1967                         ivsize = HIFN_DES_KEY_LENGTH;
1968                 else if (type == ACRYPTO_TYPE_3DES)
1969                         ivsize = HIFN_3DES_KEY_LENGTH;
1970         }
1971
1972         if (ctx->keysize != 16 && type == ACRYPTO_TYPE_AES_128) {
1973                 if (ctx->keysize == 24)
1974                         type = ACRYPTO_TYPE_AES_192;
1975                 else if (ctx->keysize == 32)
1976                         type = ACRYPTO_TYPE_AES_256;
1977         }
1978
1979         ctx->op = op;
1980         ctx->mode = mode;
1981         ctx->type = type;
1982         ctx->iv = req->info;
1983         ctx->ivsize = ivsize;
1984
1985         /*
1986          * HEAVY TODO: needs to kick Herbert XU to write documentation.
1987          * HEAVY TODO: needs to kick Herbert XU to write documentation.
1988          * HEAVY TODO: needs to kick Herbert XU to write documentation.
1989          */
1990
1991         return hifn_handle_req(req);
1992 }
1993
1994 static int hifn_process_queue(struct hifn_device *dev)
1995 {
1996         struct crypto_async_request *async_req;
1997         struct hifn_context *ctx;
1998         struct ablkcipher_request *req;
1999         unsigned long flags;
2000         int err = 0;
2001
2002         while (dev->started < HIFN_QUEUE_LENGTH) {
2003                 spin_lock_irqsave(&dev->lock, flags);
2004                 async_req = crypto_dequeue_request(&dev->queue);
2005                 spin_unlock_irqrestore(&dev->lock, flags);
2006
2007                 if (!async_req)
2008                         break;
2009
2010                 ctx = crypto_tfm_ctx(async_req->tfm);
2011                 req = container_of(async_req, struct ablkcipher_request, base);
2012
2013                 err = hifn_handle_req(req);
2014                 if (err)
2015                         break;
2016         }
2017
2018         return err;
2019 }
2020
2021 static int hifn_setup_crypto(struct ablkcipher_request *req, u8 op,
2022                 u8 type, u8 mode)
2023 {
2024         int err;
2025         struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm);
2026         struct hifn_device *dev = ctx->dev;
2027
2028         err = hifn_setup_crypto_req(req, op, type, mode);
2029         if (err)
2030                 return err;
2031
2032         if (dev->started < HIFN_QUEUE_LENGTH && dev->queue.qlen)
2033                 err = hifn_process_queue(dev);
2034
2035         return err;
2036 }
2037
2038 /*
2039  * AES ecryption functions.
2040  */
2041 static inline int hifn_encrypt_aes_ecb(struct ablkcipher_request *req)
2042 {
2043         return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2044                         ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_ECB);
2045 }
2046 static inline int hifn_encrypt_aes_cbc(struct ablkcipher_request *req)
2047 {
2048         return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2049                         ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_CBC);
2050 }
2051 static inline int hifn_encrypt_aes_cfb(struct ablkcipher_request *req)
2052 {
2053         return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2054                         ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_CFB);
2055 }
2056 static inline int hifn_encrypt_aes_ofb(struct ablkcipher_request *req)
2057 {
2058         return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2059                         ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_OFB);
2060 }
2061
2062 /*
2063  * AES decryption functions.
2064  */
2065 static inline int hifn_decrypt_aes_ecb(struct ablkcipher_request *req)
2066 {
2067         return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2068                         ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_ECB);
2069 }
2070 static inline int hifn_decrypt_aes_cbc(struct ablkcipher_request *req)
2071 {
2072         return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2073                         ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_CBC);
2074 }
2075 static inline int hifn_decrypt_aes_cfb(struct ablkcipher_request *req)
2076 {
2077         return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2078                         ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_CFB);
2079 }
2080 static inline int hifn_decrypt_aes_ofb(struct ablkcipher_request *req)
2081 {
2082         return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2083                         ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_OFB);
2084 }
2085
2086 /*
2087  * DES ecryption functions.
2088  */
2089 static inline int hifn_encrypt_des_ecb(struct ablkcipher_request *req)
2090 {
2091         return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2092                         ACRYPTO_TYPE_DES, ACRYPTO_MODE_ECB);
2093 }
2094 static inline int hifn_encrypt_des_cbc(struct ablkcipher_request *req)
2095 {
2096         return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2097                         ACRYPTO_TYPE_DES, ACRYPTO_MODE_CBC);
2098 }
2099 static inline int hifn_encrypt_des_cfb(struct ablkcipher_request *req)
2100 {
2101         return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2102                         ACRYPTO_TYPE_DES, ACRYPTO_MODE_CFB);
2103 }
2104 static inline int hifn_encrypt_des_ofb(struct ablkcipher_request *req)
2105 {
2106         return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2107                         ACRYPTO_TYPE_DES, ACRYPTO_MODE_OFB);
2108 }
2109
2110 /*
2111  * DES decryption functions.
2112  */
2113 static inline int hifn_decrypt_des_ecb(struct ablkcipher_request *req)
2114 {
2115         return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2116                         ACRYPTO_TYPE_DES, ACRYPTO_MODE_ECB);
2117 }
2118 static inline int hifn_decrypt_des_cbc(struct ablkcipher_request *req)
2119 {
2120         return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2121                         ACRYPTO_TYPE_DES, ACRYPTO_MODE_CBC);
2122 }
2123 static inline int hifn_decrypt_des_cfb(struct ablkcipher_request *req)
2124 {
2125         return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2126                         ACRYPTO_TYPE_DES, ACRYPTO_MODE_CFB);
2127 }
2128 static inline int hifn_decrypt_des_ofb(struct ablkcipher_request *req)
2129 {
2130         return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2131                         ACRYPTO_TYPE_DES, ACRYPTO_MODE_OFB);
2132 }
2133
2134 /*
2135  * 3DES ecryption functions.
2136  */
2137 static inline int hifn_encrypt_3des_ecb(struct ablkcipher_request *req)
2138 {
2139         return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2140                         ACRYPTO_TYPE_3DES, ACRYPTO_MODE_ECB);
2141 }
2142 static inline int hifn_encrypt_3des_cbc(struct ablkcipher_request *req)
2143 {
2144         return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2145                         ACRYPTO_TYPE_3DES, ACRYPTO_MODE_CBC);
2146 }
2147 static inline int hifn_encrypt_3des_cfb(struct ablkcipher_request *req)
2148 {
2149         return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2150                         ACRYPTO_TYPE_3DES, ACRYPTO_MODE_CFB);
2151 }
2152 static inline int hifn_encrypt_3des_ofb(struct ablkcipher_request *req)
2153 {
2154         return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2155                         ACRYPTO_TYPE_3DES, ACRYPTO_MODE_OFB);
2156 }
2157
2158 /*
2159  * 3DES decryption functions.
2160  */
2161 static inline int hifn_decrypt_3des_ecb(struct ablkcipher_request *req)
2162 {
2163         return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2164                         ACRYPTO_TYPE_3DES, ACRYPTO_MODE_ECB);
2165 }
2166 static inline int hifn_decrypt_3des_cbc(struct ablkcipher_request *req)
2167 {
2168         return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2169                         ACRYPTO_TYPE_3DES, ACRYPTO_MODE_CBC);
2170 }
2171 static inline int hifn_decrypt_3des_cfb(struct ablkcipher_request *req)
2172 {
2173         return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2174                         ACRYPTO_TYPE_3DES, ACRYPTO_MODE_CFB);
2175 }
2176 static inline int hifn_decrypt_3des_ofb(struct ablkcipher_request *req)
2177 {
2178         return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2179                         ACRYPTO_TYPE_3DES, ACRYPTO_MODE_OFB);
2180 }
2181
2182 struct hifn_alg_template
2183 {
2184         char name[CRYPTO_MAX_ALG_NAME];
2185         char drv_name[CRYPTO_MAX_ALG_NAME];
2186         unsigned int bsize;
2187         struct ablkcipher_alg ablkcipher;
2188 };
2189
2190 static struct hifn_alg_template hifn_alg_templates[] = {
2191         /*
2192          * 3DES ECB, CBC, CFB and OFB modes.
2193          */
2194         {
2195                 .name = "cfb(des3_ede)", .drv_name = "hifn-3des", .bsize = 8,
2196                 .ablkcipher = {
2197                         .min_keysize    =       HIFN_3DES_KEY_LENGTH,
2198                         .max_keysize    =       HIFN_3DES_KEY_LENGTH,
2199                         .setkey         =       hifn_setkey,
2200                         .encrypt        =       hifn_encrypt_3des_cfb,
2201                         .decrypt        =       hifn_decrypt_3des_cfb,
2202                 },
2203         },
2204         {
2205                 .name = "ofb(des3_ede)", .drv_name = "hifn-3des", .bsize = 8,
2206                 .ablkcipher = {
2207                         .min_keysize    =       HIFN_3DES_KEY_LENGTH,
2208                         .max_keysize    =       HIFN_3DES_KEY_LENGTH,
2209                         .setkey         =       hifn_setkey,
2210                         .encrypt        =       hifn_encrypt_3des_ofb,
2211                         .decrypt        =       hifn_decrypt_3des_ofb,
2212                 },
2213         },
2214         {
2215                 .name = "cbc(des3_ede)", .drv_name = "hifn-3des", .bsize = 8,
2216                 .ablkcipher = {
2217                         .min_keysize    =       HIFN_3DES_KEY_LENGTH,
2218                         .max_keysize    =       HIFN_3DES_KEY_LENGTH,
2219                         .setkey         =       hifn_setkey,
2220                         .encrypt        =       hifn_encrypt_3des_cbc,
2221                         .decrypt        =       hifn_decrypt_3des_cbc,
2222                 },
2223         },
2224         {
2225                 .name = "ecb(des3_ede)", .drv_name = "hifn-3des", .bsize = 8,
2226                 .ablkcipher = {
2227                         .min_keysize    =       HIFN_3DES_KEY_LENGTH,
2228                         .max_keysize    =       HIFN_3DES_KEY_LENGTH,
2229                         .setkey         =       hifn_setkey,
2230                         .encrypt        =       hifn_encrypt_3des_ecb,
2231                         .decrypt        =       hifn_decrypt_3des_ecb,
2232                 },
2233         },
2234
2235         /*
2236          * DES ECB, CBC, CFB and OFB modes.
2237          */
2238         {
2239                 .name = "cfb(des)", .drv_name = "hifn-des", .bsize = 8,
2240                 .ablkcipher = {
2241                         .min_keysize    =       HIFN_DES_KEY_LENGTH,
2242                         .max_keysize    =       HIFN_DES_KEY_LENGTH,
2243                         .setkey         =       hifn_setkey,
2244                         .encrypt        =       hifn_encrypt_des_cfb,
2245                         .decrypt        =       hifn_decrypt_des_cfb,
2246                 },
2247         },
2248         {
2249                 .name = "ofb(des)", .drv_name = "hifn-des", .bsize = 8,
2250                 .ablkcipher = {
2251                         .min_keysize    =       HIFN_DES_KEY_LENGTH,
2252                         .max_keysize    =       HIFN_DES_KEY_LENGTH,
2253                         .setkey         =       hifn_setkey,
2254                         .encrypt        =       hifn_encrypt_des_ofb,
2255                         .decrypt        =       hifn_decrypt_des_ofb,
2256                 },
2257         },
2258         {
2259                 .name = "cbc(des)", .drv_name = "hifn-des", .bsize = 8,
2260                 .ablkcipher = {
2261                         .min_keysize    =       HIFN_DES_KEY_LENGTH,
2262                         .max_keysize    =       HIFN_DES_KEY_LENGTH,
2263                         .setkey         =       hifn_setkey,
2264                         .encrypt        =       hifn_encrypt_des_cbc,
2265                         .decrypt        =       hifn_decrypt_des_cbc,
2266                 },
2267         },
2268         {
2269                 .name = "ecb(des)", .drv_name = "hifn-des", .bsize = 8,
2270                 .ablkcipher = {
2271                         .min_keysize    =       HIFN_DES_KEY_LENGTH,
2272                         .max_keysize    =       HIFN_DES_KEY_LENGTH,
2273                         .setkey         =       hifn_setkey,
2274                         .encrypt        =       hifn_encrypt_des_ecb,
2275                         .decrypt        =       hifn_decrypt_des_ecb,
2276                 },
2277         },
2278
2279         /*
2280          * AES ECB, CBC, CFB and OFB modes.
2281          */
2282         {
2283                 .name = "ecb(aes)", .drv_name = "hifn-aes", .bsize = 16,
2284                 .ablkcipher = {
2285                         .min_keysize    =       AES_MIN_KEY_SIZE,
2286                         .max_keysize    =       AES_MAX_KEY_SIZE,
2287                         .setkey         =       hifn_setkey,
2288                         .encrypt        =       hifn_encrypt_aes_ecb,
2289                         .decrypt        =       hifn_decrypt_aes_ecb,
2290                 },
2291         },
2292         {
2293                 .name = "cbc(aes)", .drv_name = "hifn-aes", .bsize = 16,
2294                 .ablkcipher = {
2295                         .min_keysize    =       AES_MIN_KEY_SIZE,
2296                         .max_keysize    =       AES_MAX_KEY_SIZE,
2297                         .setkey         =       hifn_setkey,
2298                         .encrypt        =       hifn_encrypt_aes_cbc,
2299                         .decrypt        =       hifn_decrypt_aes_cbc,
2300                 },
2301         },
2302         {
2303                 .name = "cfb(aes)", .drv_name = "hifn-aes", .bsize = 16,
2304                 .ablkcipher = {
2305                         .min_keysize    =       AES_MIN_KEY_SIZE,
2306                         .max_keysize    =       AES_MAX_KEY_SIZE,
2307                         .setkey         =       hifn_setkey,
2308                         .encrypt        =       hifn_encrypt_aes_cfb,
2309                         .decrypt        =       hifn_decrypt_aes_cfb,
2310                 },
2311         },
2312         {
2313                 .name = "ofb(aes)", .drv_name = "hifn-aes", .bsize = 16,
2314                 .ablkcipher = {
2315                         .min_keysize    =       AES_MIN_KEY_SIZE,
2316                         .max_keysize    =       AES_MAX_KEY_SIZE,
2317                         .setkey         =       hifn_setkey,
2318                         .encrypt        =       hifn_encrypt_aes_ofb,
2319                         .decrypt        =       hifn_decrypt_aes_ofb,
2320                 },
2321         },
2322 };
2323
2324 static int hifn_cra_init(struct crypto_tfm *tfm)
2325 {
2326         struct crypto_alg *alg = tfm->__crt_alg;
2327         struct hifn_crypto_alg *ha = crypto_alg_to_hifn(alg);
2328         struct hifn_context *ctx = crypto_tfm_ctx(tfm);
2329
2330         ctx->dev = ha->dev;
2331
2332         return 0;
2333 }
2334
2335 static int hifn_alg_alloc(struct hifn_device *dev, struct hifn_alg_template *t)
2336 {
2337         struct hifn_crypto_alg *alg;
2338         int err;
2339
2340         alg = kzalloc(sizeof(struct hifn_crypto_alg), GFP_KERNEL);
2341         if (!alg)
2342                 return -ENOMEM;
2343
2344         snprintf(alg->alg.cra_name, CRYPTO_MAX_ALG_NAME, "%s", t->name);
2345         snprintf(alg->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", t->drv_name);
2346
2347         alg->alg.cra_priority = 300;
2348         alg->alg.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | CRYPTO_ALG_ASYNC;
2349         alg->alg.cra_blocksize = t->bsize;
2350         alg->alg.cra_ctxsize = sizeof(struct hifn_context);
2351         alg->alg.cra_alignmask = 15;
2352         if (t->bsize == 8)
2353                 alg->alg.cra_alignmask = 3;
2354         alg->alg.cra_type = &crypto_ablkcipher_type;
2355         alg->alg.cra_module = THIS_MODULE;
2356         alg->alg.cra_u.ablkcipher = t->ablkcipher;
2357         alg->alg.cra_init = hifn_cra_init;
2358
2359         alg->dev = dev;
2360
2361         list_add_tail(&alg->entry, &dev->alg_list);
2362
2363         err = crypto_register_alg(&alg->alg);
2364         if (err) {
2365                 list_del(&alg->entry);
2366                 kfree(alg);
2367         }
2368
2369         return err;
2370 }
2371
2372 static void hifn_unregister_alg(struct hifn_device *dev)
2373 {
2374         struct hifn_crypto_alg *a, *n;
2375
2376         list_for_each_entry_safe(a, n, &dev->alg_list, entry) {
2377                 list_del(&a->entry);
2378                 crypto_unregister_alg(&a->alg);
2379                 kfree(a);
2380         }
2381 }
2382
2383 static int hifn_register_alg(struct hifn_device *dev)
2384 {
2385         int i, err;
2386
2387         for (i=0; i<ARRAY_SIZE(hifn_alg_templates); ++i) {
2388                 err = hifn_alg_alloc(dev, &hifn_alg_templates[i]);
2389                 if (err)
2390                         goto err_out_exit;
2391         }
2392
2393         return 0;
2394
2395 err_out_exit:
2396         hifn_unregister_alg(dev);
2397         return err;
2398 }
2399
2400 static int hifn_probe(struct pci_dev *pdev, const struct pci_device_id *id)
2401 {
2402         int err, i;
2403         struct hifn_device *dev;
2404         char name[8];
2405
2406         err = pci_enable_device(pdev);
2407         if (err)
2408                 return err;
2409         pci_set_master(pdev);
2410
2411         err = pci_set_dma_mask(pdev, DMA_32BIT_MASK);
2412         if (err)
2413                 goto err_out_disable_pci_device;
2414
2415         snprintf(name, sizeof(name), "hifn%d",
2416                         atomic_inc_return(&hifn_dev_number)-1);
2417
2418         err = pci_request_regions(pdev, name);
2419         if (err)
2420                 goto err_out_disable_pci_device;
2421
2422         if (pci_resource_len(pdev, 0) < HIFN_BAR0_SIZE ||
2423             pci_resource_len(pdev, 1) < HIFN_BAR1_SIZE ||
2424             pci_resource_len(pdev, 2) < HIFN_BAR2_SIZE) {
2425                 dprintk("%s: Broken hardware - I/O regions are too small.\n",
2426                                 pci_name(pdev));
2427                 err = -ENODEV;
2428                 goto err_out_free_regions;
2429         }
2430
2431         dev = kzalloc(sizeof(struct hifn_device) + sizeof(struct crypto_alg),
2432                         GFP_KERNEL);
2433         if (!dev) {
2434                 err = -ENOMEM;
2435                 goto err_out_free_regions;
2436         }
2437
2438         INIT_LIST_HEAD(&dev->alg_list);
2439
2440         snprintf(dev->name, sizeof(dev->name), "%s", name);
2441         spin_lock_init(&dev->lock);
2442
2443         for (i=0; i<3; ++i) {
2444                 unsigned long addr, size;
2445
2446                 addr = pci_resource_start(pdev, i);
2447                 size = pci_resource_len(pdev, i);
2448
2449                 dev->bar[i] = ioremap_nocache(addr, size);
2450                 if (!dev->bar[i])
2451                         goto err_out_unmap_bars;
2452         }
2453
2454         dev->result_mem = __get_free_pages(GFP_KERNEL, HIFN_MAX_RESULT_ORDER);
2455         if (!dev->result_mem) {
2456                 dprintk("Failed to allocate %d pages for result_mem.\n",
2457                                 HIFN_MAX_RESULT_ORDER);
2458                 goto err_out_unmap_bars;
2459         }
2460         memset((void *)dev->result_mem, 0, PAGE_SIZE*(1<<HIFN_MAX_RESULT_ORDER));
2461
2462         dev->dst = pci_map_single(pdev, (void *)dev->result_mem,
2463                         PAGE_SIZE << HIFN_MAX_RESULT_ORDER, PCI_DMA_FROMDEVICE);
2464
2465         dev->desc_virt = pci_alloc_consistent(pdev, sizeof(struct hifn_dma),
2466                         &dev->desc_dma);
2467         if (!dev->desc_virt) {
2468                 dprintk("Failed to allocate descriptor rings.\n");
2469                 goto err_out_free_result_pages;
2470         }
2471         memset(dev->desc_virt, 0, sizeof(struct hifn_dma));
2472
2473         dev->pdev = pdev;
2474         dev->irq = pdev->irq;
2475
2476         for (i=0; i<HIFN_D_RES_RSIZE; ++i)
2477                 dev->sa[i] = NULL;
2478
2479         pci_set_drvdata(pdev, dev);
2480
2481         crypto_init_queue(&dev->queue, 1);
2482
2483         err = request_irq(dev->irq, hifn_interrupt, IRQF_SHARED, dev->name, dev);
2484         if (err) {
2485                 dprintk("Failed to request IRQ%d: err: %d.\n", dev->irq, err);
2486                 dev->irq = 0;
2487                 goto err_out_free_desc;
2488         }
2489
2490         err = hifn_start_device(dev);
2491         if (err)
2492                 goto err_out_free_irq;
2493
2494         err = hifn_test(dev, 1, 0);
2495         if (err)
2496                 goto err_out_stop_device;
2497
2498         err = hifn_register_alg(dev);
2499         if (err)
2500                 goto err_out_stop_device;
2501
2502         INIT_DELAYED_WORK(&dev->work, hifn_work);
2503         schedule_delayed_work(&dev->work, HZ);
2504
2505         dprintk("HIFN crypto accelerator card at %s has been "
2506                         "successfully registered as %s.\n",
2507                         pci_name(pdev), dev->name);
2508
2509         return 0;
2510
2511 err_out_stop_device:
2512         hifn_reset_dma(dev, 1);
2513         hifn_stop_device(dev);
2514 err_out_free_irq:
2515         free_irq(dev->irq, dev->name);
2516 err_out_free_desc:
2517         pci_free_consistent(pdev, sizeof(struct hifn_dma),
2518                         dev->desc_virt, dev->desc_dma);
2519
2520 err_out_free_result_pages:
2521         pci_unmap_single(pdev, dev->dst, PAGE_SIZE << HIFN_MAX_RESULT_ORDER,
2522                         PCI_DMA_FROMDEVICE);
2523         free_pages(dev->result_mem, HIFN_MAX_RESULT_ORDER);
2524
2525 err_out_unmap_bars:
2526         for (i=0; i<3; ++i)
2527                 if (dev->bar[i])
2528                         iounmap(dev->bar[i]);
2529
2530 err_out_free_regions:
2531         pci_release_regions(pdev);
2532
2533 err_out_disable_pci_device:
2534         pci_disable_device(pdev);
2535
2536         return err;
2537 }
2538
2539 static void hifn_remove(struct pci_dev *pdev)
2540 {
2541         int i;
2542         struct hifn_device *dev;
2543
2544         dev = pci_get_drvdata(pdev);
2545
2546         if (dev) {
2547                 cancel_delayed_work(&dev->work);
2548                 flush_scheduled_work();
2549
2550                 hifn_unregister_alg(dev);
2551                 hifn_reset_dma(dev, 1);
2552                 hifn_stop_device(dev);
2553
2554                 free_irq(dev->irq, dev->name);
2555
2556                 hifn_flush(dev);
2557
2558                 pci_free_consistent(pdev, sizeof(struct hifn_dma),
2559                                 dev->desc_virt, dev->desc_dma);
2560                 pci_unmap_single(pdev, dev->dst,
2561                                 PAGE_SIZE << HIFN_MAX_RESULT_ORDER,
2562                                 PCI_DMA_FROMDEVICE);
2563                 free_pages(dev->result_mem, HIFN_MAX_RESULT_ORDER);
2564                 for (i=0; i<3; ++i)
2565                         if (dev->bar[i])
2566                                 iounmap(dev->bar[i]);
2567
2568                 kfree(dev);
2569         }
2570
2571         pci_release_regions(pdev);
2572         pci_disable_device(pdev);
2573 }
2574
2575 static struct pci_device_id hifn_pci_tbl[] = {
2576         { PCI_DEVICE(PCI_VENDOR_ID_HIFN, PCI_DEVICE_ID_HIFN_7955) },
2577         { PCI_DEVICE(PCI_VENDOR_ID_HIFN, PCI_DEVICE_ID_HIFN_7956) },
2578         { 0 }
2579 };
2580 MODULE_DEVICE_TABLE(pci, hifn_pci_tbl);
2581
2582 static struct pci_driver hifn_pci_driver = {
2583         .name     = "hifn795x",
2584         .id_table = hifn_pci_tbl,
2585         .probe    = hifn_probe,
2586         .remove   = __devexit_p(hifn_remove),
2587 };
2588
2589 static int __devinit hifn_init(void)
2590 {
2591         int err;
2592
2593         err = pci_register_driver(&hifn_pci_driver);
2594         if (err < 0) {
2595                 dprintk("Failed to register PCI driver for %s device.\n",
2596                                 hifn_pci_driver.name);
2597                 return -ENODEV;
2598         }
2599
2600         printk(KERN_INFO "Driver for HIFN 795x crypto accelerator chip "
2601                         "has been successfully registered.\n");
2602
2603         return 0;
2604 }
2605
2606 static void __devexit hifn_fini(void)
2607 {
2608         pci_unregister_driver(&hifn_pci_driver);
2609
2610         printk(KERN_INFO "Driver for HIFN 795x crypto accelerator chip "
2611                         "has been successfully unregistered.\n");
2612 }
2613
2614 module_init(hifn_init);
2615 module_exit(hifn_fini);
2616
2617 MODULE_LICENSE("GPL");
2618 MODULE_AUTHOR("Evgeniy Polyakov <johnpol@2ka.mipt.ru>");
2619 MODULE_DESCRIPTION("Driver for HIFN 795x crypto accelerator chip.");