blob: 6ace6dd5a239c6c1995e065b6745bc6082a5e0d7 [file] [log] [blame]
Gary R Hook4b394a22016-07-26 19:10:21 -05001/*
2 * AMD Cryptographic Coprocessor (CCP) driver
3 *
Gary R Hook68cc6522017-07-17 15:00:49 -05004 * Copyright (C) 2016,2017 Advanced Micro Devices, Inc.
Gary R Hook4b394a22016-07-26 19:10:21 -05005 *
6 * Author: Gary R Hook <gary.hook@amd.com>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
11 */
12
13#include <linux/module.h>
14#include <linux/kernel.h>
15#include <linux/pci.h>
16#include <linux/kthread.h>
Gary R Hook3cdbe342017-05-02 17:33:40 -050017#include <linux/debugfs.h>
Gary R Hook4b394a22016-07-26 19:10:21 -050018#include <linux/dma-mapping.h>
19#include <linux/interrupt.h>
20#include <linux/compiler.h>
21#include <linux/ccp.h>
22
23#include "ccp-dev.h"
24
Gary R Hook103600a2016-10-18 17:33:37 -050025/* Allocate the requested number of contiguous LSB slots
26 * from the LSB bitmap. Look in the private range for this
27 * queue first; failing that, check the public area.
28 * If no space is available, wait around.
29 * Return: first slot number
30 */
Gary R Hook4b394a22016-07-26 19:10:21 -050031static u32 ccp_lsb_alloc(struct ccp_cmd_queue *cmd_q, unsigned int count)
32{
33 struct ccp_device *ccp;
34 int start;
35
36 /* First look at the map for the queue */
37 if (cmd_q->lsb >= 0) {
38 start = (u32)bitmap_find_next_zero_area(cmd_q->lsbmap,
39 LSB_SIZE,
40 0, count, 0);
41 if (start < LSB_SIZE) {
42 bitmap_set(cmd_q->lsbmap, start, count);
43 return start + cmd_q->lsb * LSB_SIZE;
44 }
45 }
46
47 /* No joy; try to get an entry from the shared blocks */
48 ccp = cmd_q->ccp;
49 for (;;) {
50 mutex_lock(&ccp->sb_mutex);
51
52 start = (u32)bitmap_find_next_zero_area(ccp->lsbmap,
53 MAX_LSB_CNT * LSB_SIZE,
54 0,
55 count, 0);
56 if (start <= MAX_LSB_CNT * LSB_SIZE) {
57 bitmap_set(ccp->lsbmap, start, count);
58
59 mutex_unlock(&ccp->sb_mutex);
Gary R Hook103600a2016-10-18 17:33:37 -050060 return start;
Gary R Hook4b394a22016-07-26 19:10:21 -050061 }
62
63 ccp->sb_avail = 0;
64
65 mutex_unlock(&ccp->sb_mutex);
66
67 /* Wait for KSB entries to become available */
68 if (wait_event_interruptible(ccp->sb_queue, ccp->sb_avail))
69 return 0;
70 }
71}
72
Gary R Hook103600a2016-10-18 17:33:37 -050073/* Free a number of LSB slots from the bitmap, starting at
74 * the indicated starting slot number.
75 */
Gary R Hook4b394a22016-07-26 19:10:21 -050076static void ccp_lsb_free(struct ccp_cmd_queue *cmd_q, unsigned int start,
77 unsigned int count)
78{
Gary R Hook4b394a22016-07-26 19:10:21 -050079 if (!start)
80 return;
81
Gary R Hook103600a2016-10-18 17:33:37 -050082 if (cmd_q->lsb == start) {
Gary R Hook4b394a22016-07-26 19:10:21 -050083 /* An entry from the private LSB */
Gary R Hook103600a2016-10-18 17:33:37 -050084 bitmap_clear(cmd_q->lsbmap, start, count);
Gary R Hook4b394a22016-07-26 19:10:21 -050085 } else {
86 /* From the shared LSBs */
87 struct ccp_device *ccp = cmd_q->ccp;
88
89 mutex_lock(&ccp->sb_mutex);
90 bitmap_clear(ccp->lsbmap, start, count);
91 ccp->sb_avail = 1;
92 mutex_unlock(&ccp->sb_mutex);
93 wake_up_interruptible_all(&ccp->sb_queue);
94 }
95}
96
97/* CCP version 5: Union to define the function field (cmd_reg1/dword0) */
98union ccp_function {
99 struct {
100 u16 size:7;
101 u16 encrypt:1;
102 u16 mode:5;
103 u16 type:2;
104 } aes;
105 struct {
106 u16 size:7;
107 u16 encrypt:1;
108 u16 rsvd:5;
109 u16 type:2;
110 } aes_xts;
111 struct {
Gary R Hook990672d2017-03-15 13:20:52 -0500112 u16 size:7;
113 u16 encrypt:1;
114 u16 mode:5;
115 u16 type:2;
116 } des3;
117 struct {
Gary R Hook4b394a22016-07-26 19:10:21 -0500118 u16 rsvd1:10;
119 u16 type:4;
120 u16 rsvd2:1;
121 } sha;
122 struct {
123 u16 mode:3;
124 u16 size:12;
125 } rsa;
126 struct {
127 u16 byteswap:2;
128 u16 bitwise:3;
129 u16 reflect:2;
130 u16 rsvd:8;
131 } pt;
132 struct {
133 u16 rsvd:13;
134 } zlib;
135 struct {
136 u16 size:10;
137 u16 type:2;
138 u16 mode:3;
139 } ecc;
140 u16 raw;
141};
142
143#define CCP_AES_SIZE(p) ((p)->aes.size)
144#define CCP_AES_ENCRYPT(p) ((p)->aes.encrypt)
145#define CCP_AES_MODE(p) ((p)->aes.mode)
146#define CCP_AES_TYPE(p) ((p)->aes.type)
147#define CCP_XTS_SIZE(p) ((p)->aes_xts.size)
148#define CCP_XTS_ENCRYPT(p) ((p)->aes_xts.encrypt)
Gary R Hook990672d2017-03-15 13:20:52 -0500149#define CCP_DES3_SIZE(p) ((p)->des3.size)
150#define CCP_DES3_ENCRYPT(p) ((p)->des3.encrypt)
151#define CCP_DES3_MODE(p) ((p)->des3.mode)
152#define CCP_DES3_TYPE(p) ((p)->des3.type)
Gary R Hook4b394a22016-07-26 19:10:21 -0500153#define CCP_SHA_TYPE(p) ((p)->sha.type)
154#define CCP_RSA_SIZE(p) ((p)->rsa.size)
155#define CCP_PT_BYTESWAP(p) ((p)->pt.byteswap)
156#define CCP_PT_BITWISE(p) ((p)->pt.bitwise)
157#define CCP_ECC_MODE(p) ((p)->ecc.mode)
158#define CCP_ECC_AFFINE(p) ((p)->ecc.one)
159
160/* Word 0 */
161#define CCP5_CMD_DW0(p) ((p)->dw0)
162#define CCP5_CMD_SOC(p) (CCP5_CMD_DW0(p).soc)
163#define CCP5_CMD_IOC(p) (CCP5_CMD_DW0(p).ioc)
164#define CCP5_CMD_INIT(p) (CCP5_CMD_DW0(p).init)
165#define CCP5_CMD_EOM(p) (CCP5_CMD_DW0(p).eom)
166#define CCP5_CMD_FUNCTION(p) (CCP5_CMD_DW0(p).function)
167#define CCP5_CMD_ENGINE(p) (CCP5_CMD_DW0(p).engine)
168#define CCP5_CMD_PROT(p) (CCP5_CMD_DW0(p).prot)
169
170/* Word 1 */
171#define CCP5_CMD_DW1(p) ((p)->length)
172#define CCP5_CMD_LEN(p) (CCP5_CMD_DW1(p))
173
174/* Word 2 */
175#define CCP5_CMD_DW2(p) ((p)->src_lo)
176#define CCP5_CMD_SRC_LO(p) (CCP5_CMD_DW2(p))
177
178/* Word 3 */
179#define CCP5_CMD_DW3(p) ((p)->dw3)
180#define CCP5_CMD_SRC_MEM(p) ((p)->dw3.src_mem)
181#define CCP5_CMD_SRC_HI(p) ((p)->dw3.src_hi)
182#define CCP5_CMD_LSB_ID(p) ((p)->dw3.lsb_cxt_id)
183#define CCP5_CMD_FIX_SRC(p) ((p)->dw3.fixed)
184
185/* Words 4/5 */
186#define CCP5_CMD_DW4(p) ((p)->dw4)
187#define CCP5_CMD_DST_LO(p) (CCP5_CMD_DW4(p).dst_lo)
188#define CCP5_CMD_DW5(p) ((p)->dw5.fields.dst_hi)
189#define CCP5_CMD_DST_HI(p) (CCP5_CMD_DW5(p))
190#define CCP5_CMD_DST_MEM(p) ((p)->dw5.fields.dst_mem)
191#define CCP5_CMD_FIX_DST(p) ((p)->dw5.fields.fixed)
192#define CCP5_CMD_SHA_LO(p) ((p)->dw4.sha_len_lo)
193#define CCP5_CMD_SHA_HI(p) ((p)->dw5.sha_len_hi)
194
195/* Word 6/7 */
196#define CCP5_CMD_DW6(p) ((p)->key_lo)
197#define CCP5_CMD_KEY_LO(p) (CCP5_CMD_DW6(p))
198#define CCP5_CMD_DW7(p) ((p)->dw7)
199#define CCP5_CMD_KEY_HI(p) ((p)->dw7.key_hi)
200#define CCP5_CMD_KEY_MEM(p) ((p)->dw7.key_mem)
201
202static inline u32 low_address(unsigned long addr)
203{
204 return (u64)addr & 0x0ffffffff;
205}
206
207static inline u32 high_address(unsigned long addr)
208{
209 return ((u64)addr >> 32) & 0x00000ffff;
210}
211
212static unsigned int ccp5_get_free_slots(struct ccp_cmd_queue *cmd_q)
213{
214 unsigned int head_idx, n;
215 u32 head_lo, queue_start;
216
217 queue_start = low_address(cmd_q->qdma_tail);
218 head_lo = ioread32(cmd_q->reg_head_lo);
219 head_idx = (head_lo - queue_start) / sizeof(struct ccp5_desc);
220
221 n = head_idx + COMMANDS_PER_QUEUE - cmd_q->qidx - 1;
222
223 return n % COMMANDS_PER_QUEUE; /* Always one unused spot */
224}
225
226static int ccp5_do_cmd(struct ccp5_desc *desc,
227 struct ccp_cmd_queue *cmd_q)
228{
229 u32 *mP;
230 __le32 *dP;
231 u32 tail;
232 int i;
233 int ret = 0;
234
Gary R Hook3cdbe342017-05-02 17:33:40 -0500235 cmd_q->total_ops++;
236
Gary R Hook4b394a22016-07-26 19:10:21 -0500237 if (CCP5_CMD_SOC(desc)) {
238 CCP5_CMD_IOC(desc) = 1;
239 CCP5_CMD_SOC(desc) = 0;
240 }
241 mutex_lock(&cmd_q->q_mutex);
242
243 mP = (u32 *) &cmd_q->qbase[cmd_q->qidx];
244 dP = (__le32 *) desc;
245 for (i = 0; i < 8; i++)
246 mP[i] = cpu_to_le32(dP[i]); /* handle endianness */
247
248 cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
249
250 /* The data used by this command must be flushed to memory */
251 wmb();
252
253 /* Write the new tail address back to the queue register */
254 tail = low_address(cmd_q->qdma_tail + cmd_q->qidx * Q_DESC_SIZE);
255 iowrite32(tail, cmd_q->reg_tail_lo);
256
257 /* Turn the queue back on using our cached control register */
258 iowrite32(cmd_q->qcontrol | CMD5_Q_RUN, cmd_q->reg_control);
259 mutex_unlock(&cmd_q->q_mutex);
260
261 if (CCP5_CMD_IOC(desc)) {
262 /* Wait for the job to complete */
263 ret = wait_event_interruptible(cmd_q->int_queue,
264 cmd_q->int_rcvd);
265 if (ret || cmd_q->cmd_error) {
Gary R Hook4cdf1012017-02-09 15:49:57 -0600266 /* Log the error and flush the queue by
267 * moving the head pointer
268 */
Gary R Hook81422ba2016-09-28 11:53:56 -0500269 if (cmd_q->cmd_error)
270 ccp_log_error(cmd_q->ccp,
271 cmd_q->cmd_error);
Gary R Hook4cdf1012017-02-09 15:49:57 -0600272 iowrite32(tail, cmd_q->reg_head_lo);
Gary R Hook4b394a22016-07-26 19:10:21 -0500273 if (!ret)
274 ret = -EIO;
275 }
276 cmd_q->int_rcvd = 0;
277 }
278
Gary R Hook4cdf1012017-02-09 15:49:57 -0600279 return ret;
Gary R Hook4b394a22016-07-26 19:10:21 -0500280}
281
282static int ccp5_perform_aes(struct ccp_op *op)
283{
284 struct ccp5_desc desc;
285 union ccp_function function;
286 u32 key_addr = op->sb_key * LSB_ITEM_SIZE;
287
Gary R Hook3cdbe342017-05-02 17:33:40 -0500288 op->cmd_q->total_aes_ops++;
289
Gary R Hook4b394a22016-07-26 19:10:21 -0500290 /* Zero out all the fields of the command desc */
291 memset(&desc, 0, Q_DESC_SIZE);
292
293 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_AES;
294
295 CCP5_CMD_SOC(&desc) = op->soc;
296 CCP5_CMD_IOC(&desc) = 1;
297 CCP5_CMD_INIT(&desc) = op->init;
298 CCP5_CMD_EOM(&desc) = op->eom;
299 CCP5_CMD_PROT(&desc) = 0;
300
301 function.raw = 0;
302 CCP_AES_ENCRYPT(&function) = op->u.aes.action;
303 CCP_AES_MODE(&function) = op->u.aes.mode;
304 CCP_AES_TYPE(&function) = op->u.aes.type;
Gary R Hookf7cc02b32017-02-08 13:07:06 -0600305 CCP_AES_SIZE(&function) = op->u.aes.size;
Gary R Hook4b394a22016-07-26 19:10:21 -0500306
307 CCP5_CMD_FUNCTION(&desc) = function.raw;
308
309 CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
310
311 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
312 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
313 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
314
315 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
316 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
317 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
318
319 CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr);
320 CCP5_CMD_KEY_HI(&desc) = 0;
321 CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB;
322 CCP5_CMD_LSB_ID(&desc) = op->sb_ctx;
323
324 return ccp5_do_cmd(&desc, op->cmd_q);
325}
326
327static int ccp5_perform_xts_aes(struct ccp_op *op)
328{
329 struct ccp5_desc desc;
330 union ccp_function function;
331 u32 key_addr = op->sb_key * LSB_ITEM_SIZE;
332
Gary R Hook3cdbe342017-05-02 17:33:40 -0500333 op->cmd_q->total_xts_aes_ops++;
334
Gary R Hook4b394a22016-07-26 19:10:21 -0500335 /* Zero out all the fields of the command desc */
336 memset(&desc, 0, Q_DESC_SIZE);
337
338 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_XTS_AES_128;
339
340 CCP5_CMD_SOC(&desc) = op->soc;
341 CCP5_CMD_IOC(&desc) = 1;
342 CCP5_CMD_INIT(&desc) = op->init;
343 CCP5_CMD_EOM(&desc) = op->eom;
344 CCP5_CMD_PROT(&desc) = 0;
345
346 function.raw = 0;
347 CCP_XTS_ENCRYPT(&function) = op->u.xts.action;
348 CCP_XTS_SIZE(&function) = op->u.xts.unit_size;
349 CCP5_CMD_FUNCTION(&desc) = function.raw;
350
351 CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
352
353 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
354 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
355 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
356
357 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
358 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
359 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
360
361 CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr);
362 CCP5_CMD_KEY_HI(&desc) = 0;
363 CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB;
364 CCP5_CMD_LSB_ID(&desc) = op->sb_ctx;
365
366 return ccp5_do_cmd(&desc, op->cmd_q);
367}
368
369static int ccp5_perform_sha(struct ccp_op *op)
370{
371 struct ccp5_desc desc;
372 union ccp_function function;
373
Gary R Hook3cdbe342017-05-02 17:33:40 -0500374 op->cmd_q->total_sha_ops++;
375
Gary R Hook4b394a22016-07-26 19:10:21 -0500376 /* Zero out all the fields of the command desc */
377 memset(&desc, 0, Q_DESC_SIZE);
378
379 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_SHA;
380
381 CCP5_CMD_SOC(&desc) = op->soc;
382 CCP5_CMD_IOC(&desc) = 1;
383 CCP5_CMD_INIT(&desc) = 1;
384 CCP5_CMD_EOM(&desc) = op->eom;
385 CCP5_CMD_PROT(&desc) = 0;
386
387 function.raw = 0;
388 CCP_SHA_TYPE(&function) = op->u.sha.type;
389 CCP5_CMD_FUNCTION(&desc) = function.raw;
390
391 CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
392
393 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
394 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
395 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
396
397 CCP5_CMD_LSB_ID(&desc) = op->sb_ctx;
398
399 if (op->eom) {
400 CCP5_CMD_SHA_LO(&desc) = lower_32_bits(op->u.sha.msg_bits);
401 CCP5_CMD_SHA_HI(&desc) = upper_32_bits(op->u.sha.msg_bits);
402 } else {
403 CCP5_CMD_SHA_LO(&desc) = 0;
404 CCP5_CMD_SHA_HI(&desc) = 0;
405 }
406
407 return ccp5_do_cmd(&desc, op->cmd_q);
408}
409
Gary R Hook990672d2017-03-15 13:20:52 -0500410static int ccp5_perform_des3(struct ccp_op *op)
411{
412 struct ccp5_desc desc;
413 union ccp_function function;
414 u32 key_addr = op->sb_key * LSB_ITEM_SIZE;
415
Gary R Hook3cdbe342017-05-02 17:33:40 -0500416 op->cmd_q->total_3des_ops++;
417
Gary R Hook990672d2017-03-15 13:20:52 -0500418 /* Zero out all the fields of the command desc */
419 memset(&desc, 0, sizeof(struct ccp5_desc));
420
421 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_DES3;
422
423 CCP5_CMD_SOC(&desc) = op->soc;
424 CCP5_CMD_IOC(&desc) = 1;
425 CCP5_CMD_INIT(&desc) = op->init;
426 CCP5_CMD_EOM(&desc) = op->eom;
427 CCP5_CMD_PROT(&desc) = 0;
428
429 function.raw = 0;
430 CCP_DES3_ENCRYPT(&function) = op->u.des3.action;
431 CCP_DES3_MODE(&function) = op->u.des3.mode;
432 CCP_DES3_TYPE(&function) = op->u.des3.type;
Gary R Hook51de7dd2017-03-28 08:58:28 -0500433 CCP5_CMD_FUNCTION(&desc) = function.raw;
Gary R Hook990672d2017-03-15 13:20:52 -0500434
Gary R Hook51de7dd2017-03-28 08:58:28 -0500435 CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
Gary R Hook990672d2017-03-15 13:20:52 -0500436
Gary R Hook51de7dd2017-03-28 08:58:28 -0500437 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
438 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
439 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
Gary R Hook990672d2017-03-15 13:20:52 -0500440
Gary R Hook51de7dd2017-03-28 08:58:28 -0500441 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
442 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
443 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
Gary R Hook990672d2017-03-15 13:20:52 -0500444
Gary R Hook51de7dd2017-03-28 08:58:28 -0500445 CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr);
Gary R Hook990672d2017-03-15 13:20:52 -0500446 CCP5_CMD_KEY_HI(&desc) = 0;
Gary R Hook51de7dd2017-03-28 08:58:28 -0500447 CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB;
448 CCP5_CMD_LSB_ID(&desc) = op->sb_ctx;
Gary R Hook990672d2017-03-15 13:20:52 -0500449
450 return ccp5_do_cmd(&desc, op->cmd_q);
451}
452
Gary R Hook4b394a22016-07-26 19:10:21 -0500453static int ccp5_perform_rsa(struct ccp_op *op)
454{
455 struct ccp5_desc desc;
456 union ccp_function function;
457
Gary R Hook3cdbe342017-05-02 17:33:40 -0500458 op->cmd_q->total_rsa_ops++;
459
Gary R Hook4b394a22016-07-26 19:10:21 -0500460 /* Zero out all the fields of the command desc */
461 memset(&desc, 0, Q_DESC_SIZE);
462
463 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_RSA;
464
465 CCP5_CMD_SOC(&desc) = op->soc;
466 CCP5_CMD_IOC(&desc) = 1;
467 CCP5_CMD_INIT(&desc) = 0;
468 CCP5_CMD_EOM(&desc) = 1;
469 CCP5_CMD_PROT(&desc) = 0;
470
471 function.raw = 0;
Gary R Hook6ba46c72017-07-17 15:16:13 -0500472 CCP_RSA_SIZE(&function) = (op->u.rsa.mod_size + 7) >> 3;
Gary R Hook4b394a22016-07-26 19:10:21 -0500473 CCP5_CMD_FUNCTION(&desc) = function.raw;
474
475 CCP5_CMD_LEN(&desc) = op->u.rsa.input_len;
476
477 /* Source is from external memory */
478 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
479 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
480 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
481
482 /* Destination is in external memory */
483 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
484 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
485 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
486
Gary R Hook6ba46c72017-07-17 15:16:13 -0500487 /* Key (Exponent) is in external memory */
488 CCP5_CMD_KEY_LO(&desc) = ccp_addr_lo(&op->exp.u.dma);
489 CCP5_CMD_KEY_HI(&desc) = ccp_addr_hi(&op->exp.u.dma);
490 CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
Gary R Hook4b394a22016-07-26 19:10:21 -0500491
492 return ccp5_do_cmd(&desc, op->cmd_q);
493}
494
495static int ccp5_perform_passthru(struct ccp_op *op)
496{
497 struct ccp5_desc desc;
498 union ccp_function function;
499 struct ccp_dma_info *saddr = &op->src.u.dma;
500 struct ccp_dma_info *daddr = &op->dst.u.dma;
501
Gary R Hook990672d2017-03-15 13:20:52 -0500502
Gary R Hook3cdbe342017-05-02 17:33:40 -0500503 op->cmd_q->total_pt_ops++;
504
Gary R Hook4b394a22016-07-26 19:10:21 -0500505 memset(&desc, 0, Q_DESC_SIZE);
506
507 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_PASSTHRU;
508
509 CCP5_CMD_SOC(&desc) = 0;
510 CCP5_CMD_IOC(&desc) = 1;
511 CCP5_CMD_INIT(&desc) = 0;
512 CCP5_CMD_EOM(&desc) = op->eom;
513 CCP5_CMD_PROT(&desc) = 0;
514
515 function.raw = 0;
516 CCP_PT_BYTESWAP(&function) = op->u.passthru.byte_swap;
517 CCP_PT_BITWISE(&function) = op->u.passthru.bit_mod;
518 CCP5_CMD_FUNCTION(&desc) = function.raw;
519
520 /* Length of source data is always 256 bytes */
521 if (op->src.type == CCP_MEMTYPE_SYSTEM)
522 CCP5_CMD_LEN(&desc) = saddr->length;
523 else
524 CCP5_CMD_LEN(&desc) = daddr->length;
525
526 if (op->src.type == CCP_MEMTYPE_SYSTEM) {
527 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
528 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
529 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
530
531 if (op->u.passthru.bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
532 CCP5_CMD_LSB_ID(&desc) = op->sb_key;
533 } else {
534 u32 key_addr = op->src.u.sb * CCP_SB_BYTES;
535
536 CCP5_CMD_SRC_LO(&desc) = lower_32_bits(key_addr);
537 CCP5_CMD_SRC_HI(&desc) = 0;
538 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SB;
539 }
540
541 if (op->dst.type == CCP_MEMTYPE_SYSTEM) {
542 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
543 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
544 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
545 } else {
546 u32 key_addr = op->dst.u.sb * CCP_SB_BYTES;
547
548 CCP5_CMD_DST_LO(&desc) = lower_32_bits(key_addr);
549 CCP5_CMD_DST_HI(&desc) = 0;
550 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SB;
551 }
552
553 return ccp5_do_cmd(&desc, op->cmd_q);
554}
555
556static int ccp5_perform_ecc(struct ccp_op *op)
557{
558 struct ccp5_desc desc;
559 union ccp_function function;
560
Gary R Hook3cdbe342017-05-02 17:33:40 -0500561 op->cmd_q->total_ecc_ops++;
562
Gary R Hook4b394a22016-07-26 19:10:21 -0500563 /* Zero out all the fields of the command desc */
564 memset(&desc, 0, Q_DESC_SIZE);
565
566 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_ECC;
567
568 CCP5_CMD_SOC(&desc) = 0;
569 CCP5_CMD_IOC(&desc) = 1;
570 CCP5_CMD_INIT(&desc) = 0;
571 CCP5_CMD_EOM(&desc) = 1;
572 CCP5_CMD_PROT(&desc) = 0;
573
574 function.raw = 0;
575 function.ecc.mode = op->u.ecc.function;
576 CCP5_CMD_FUNCTION(&desc) = function.raw;
577
578 CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
579
580 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
581 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
582 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
583
584 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
585 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
586 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
587
588 return ccp5_do_cmd(&desc, op->cmd_q);
589}
590
591static int ccp_find_lsb_regions(struct ccp_cmd_queue *cmd_q, u64 status)
592{
593 int q_mask = 1 << cmd_q->id;
594 int queues = 0;
595 int j;
596
597 /* Build a bit mask to know which LSBs this queue has access to.
598 * Don't bother with segment 0 as it has special privileges.
599 */
600 for (j = 1; j < MAX_LSB_CNT; j++) {
601 if (status & q_mask)
602 bitmap_set(cmd_q->lsbmask, j, 1);
603 status >>= LSB_REGION_WIDTH;
604 }
605 queues = bitmap_weight(cmd_q->lsbmask, MAX_LSB_CNT);
Gary R Hooka60496a2017-02-09 15:49:48 -0600606 dev_dbg(cmd_q->ccp->dev, "Queue %d can access %d LSB regions\n",
Gary R Hook4b394a22016-07-26 19:10:21 -0500607 cmd_q->id, queues);
608
609 return queues ? 0 : -EINVAL;
610}
611
Gary R Hook4b394a22016-07-26 19:10:21 -0500612static int ccp_find_and_assign_lsb_to_q(struct ccp_device *ccp,
613 int lsb_cnt, int n_lsbs,
614 unsigned long *lsb_pub)
615{
616 DECLARE_BITMAP(qlsb, MAX_LSB_CNT);
617 int bitno;
618 int qlsb_wgt;
619 int i;
620
621 /* For each queue:
622 * If the count of potential LSBs available to a queue matches the
623 * ordinal given to us in lsb_cnt:
624 * Copy the mask of possible LSBs for this queue into "qlsb";
625 * For each bit in qlsb, see if the corresponding bit in the
626 * aggregation mask is set; if so, we have a match.
627 * If we have a match, clear the bit in the aggregation to
628 * mark it as no longer available.
629 * If there is no match, clear the bit in qlsb and keep looking.
630 */
631 for (i = 0; i < ccp->cmd_q_count; i++) {
632 struct ccp_cmd_queue *cmd_q = &ccp->cmd_q[i];
633
634 qlsb_wgt = bitmap_weight(cmd_q->lsbmask, MAX_LSB_CNT);
635
636 if (qlsb_wgt == lsb_cnt) {
637 bitmap_copy(qlsb, cmd_q->lsbmask, MAX_LSB_CNT);
638
639 bitno = find_first_bit(qlsb, MAX_LSB_CNT);
640 while (bitno < MAX_LSB_CNT) {
641 if (test_bit(bitno, lsb_pub)) {
642 /* We found an available LSB
643 * that this queue can access
644 */
645 cmd_q->lsb = bitno;
646 bitmap_clear(lsb_pub, bitno, 1);
Gary R Hooka60496a2017-02-09 15:49:48 -0600647 dev_dbg(ccp->dev,
Gary R Hook4b394a22016-07-26 19:10:21 -0500648 "Queue %d gets LSB %d\n",
649 i, bitno);
650 break;
651 }
652 bitmap_clear(qlsb, bitno, 1);
653 bitno = find_first_bit(qlsb, MAX_LSB_CNT);
654 }
655 if (bitno >= MAX_LSB_CNT)
656 return -EINVAL;
657 n_lsbs--;
658 }
659 }
660 return n_lsbs;
661}
662
663/* For each queue, from the most- to least-constrained:
664 * find an LSB that can be assigned to the queue. If there are N queues that
665 * can only use M LSBs, where N > M, fail; otherwise, every queue will get a
666 * dedicated LSB. Remaining LSB regions become a shared resource.
667 * If we have fewer LSBs than queues, all LSB regions become shared resources.
668 */
669static int ccp_assign_lsbs(struct ccp_device *ccp)
670{
671 DECLARE_BITMAP(lsb_pub, MAX_LSB_CNT);
672 DECLARE_BITMAP(qlsb, MAX_LSB_CNT);
673 int n_lsbs = 0;
674 int bitno;
675 int i, lsb_cnt;
676 int rc = 0;
677
678 bitmap_zero(lsb_pub, MAX_LSB_CNT);
679
680 /* Create an aggregate bitmap to get a total count of available LSBs */
681 for (i = 0; i < ccp->cmd_q_count; i++)
682 bitmap_or(lsb_pub,
683 lsb_pub, ccp->cmd_q[i].lsbmask,
684 MAX_LSB_CNT);
685
686 n_lsbs = bitmap_weight(lsb_pub, MAX_LSB_CNT);
687
688 if (n_lsbs >= ccp->cmd_q_count) {
689 /* We have enough LSBS to give every queue a private LSB.
690 * Brute force search to start with the queues that are more
691 * constrained in LSB choice. When an LSB is privately
692 * assigned, it is removed from the public mask.
693 * This is an ugly N squared algorithm with some optimization.
694 */
695 for (lsb_cnt = 1;
696 n_lsbs && (lsb_cnt <= MAX_LSB_CNT);
697 lsb_cnt++) {
698 rc = ccp_find_and_assign_lsb_to_q(ccp, lsb_cnt, n_lsbs,
699 lsb_pub);
700 if (rc < 0)
701 return -EINVAL;
702 n_lsbs = rc;
703 }
704 }
705
706 rc = 0;
707 /* What's left of the LSBs, according to the public mask, now become
708 * shared. Any zero bits in the lsb_pub mask represent an LSB region
709 * that can't be used as a shared resource, so mark the LSB slots for
710 * them as "in use".
711 */
712 bitmap_copy(qlsb, lsb_pub, MAX_LSB_CNT);
713
714 bitno = find_first_zero_bit(qlsb, MAX_LSB_CNT);
715 while (bitno < MAX_LSB_CNT) {
716 bitmap_set(ccp->lsbmap, bitno * LSB_SIZE, LSB_SIZE);
717 bitmap_set(qlsb, bitno, 1);
718 bitno = find_first_zero_bit(qlsb, MAX_LSB_CNT);
719 }
720
721 return rc;
722}
723
Gary R Hook6263b512017-04-21 10:50:14 -0500724static void ccp5_disable_queue_interrupts(struct ccp_device *ccp)
725{
726 unsigned int i;
727
728 for (i = 0; i < ccp->cmd_q_count; i++)
729 iowrite32(0x0, ccp->cmd_q[i].reg_int_enable);
730}
731
732static void ccp5_enable_queue_interrupts(struct ccp_device *ccp)
733{
734 unsigned int i;
735
736 for (i = 0; i < ccp->cmd_q_count; i++)
737 iowrite32(SUPPORTED_INTERRUPTS, ccp->cmd_q[i].reg_int_enable);
738}
739
740static void ccp5_irq_bh(unsigned long data)
741{
742 struct ccp_device *ccp = (struct ccp_device *)data;
743 u32 status;
744 unsigned int i;
745
746 for (i = 0; i < ccp->cmd_q_count; i++) {
747 struct ccp_cmd_queue *cmd_q = &ccp->cmd_q[i];
748
749 status = ioread32(cmd_q->reg_interrupt_status);
750
751 if (status) {
752 cmd_q->int_status = status;
753 cmd_q->q_status = ioread32(cmd_q->reg_status);
754 cmd_q->q_int_status = ioread32(cmd_q->reg_int_status);
755
756 /* On error, only save the first error value */
757 if ((status & INT_ERROR) && !cmd_q->cmd_error)
758 cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status);
759
760 cmd_q->int_rcvd = 1;
761
762 /* Acknowledge the interrupt and wake the kthread */
763 iowrite32(status, cmd_q->reg_interrupt_status);
764 wake_up_interruptible(&cmd_q->int_queue);
765 }
766 }
767 ccp5_enable_queue_interrupts(ccp);
768}
769
770static irqreturn_t ccp5_irq_handler(int irq, void *data)
771{
Brijesh Singh720419f2017-07-06 09:59:14 -0500772 struct ccp_device *ccp = (struct ccp_device *)data;
Gary R Hook6263b512017-04-21 10:50:14 -0500773
774 ccp5_disable_queue_interrupts(ccp);
Gary R Hook3cdbe342017-05-02 17:33:40 -0500775 ccp->total_interrupts++;
Gary R Hook6263b512017-04-21 10:50:14 -0500776 if (ccp->use_tasklet)
777 tasklet_schedule(&ccp->irq_tasklet);
778 else
779 ccp5_irq_bh((unsigned long)ccp);
780 return IRQ_HANDLED;
781}
782
Gary R Hook4b394a22016-07-26 19:10:21 -0500783static int ccp5_init(struct ccp_device *ccp)
784{
785 struct device *dev = ccp->dev;
786 struct ccp_cmd_queue *cmd_q;
787 struct dma_pool *dma_pool;
788 char dma_pool_name[MAX_DMAPOOL_NAME_LEN];
789 unsigned int qmr, qim, i;
790 u64 status;
791 u32 status_lo, status_hi;
792 int ret;
793
794 /* Find available queues */
795 qim = 0;
796 qmr = ioread32(ccp->io_regs + Q_MASK_REG);
797 for (i = 0; i < MAX_HW_QUEUES; i++) {
798
799 if (!(qmr & (1 << i)))
800 continue;
801
802 /* Allocate a dma pool for this queue */
803 snprintf(dma_pool_name, sizeof(dma_pool_name), "%s_q%d",
804 ccp->name, i);
805 dma_pool = dma_pool_create(dma_pool_name, dev,
806 CCP_DMAPOOL_MAX_SIZE,
807 CCP_DMAPOOL_ALIGN, 0);
808 if (!dma_pool) {
809 dev_err(dev, "unable to allocate dma pool\n");
810 ret = -ENOMEM;
811 }
812
813 cmd_q = &ccp->cmd_q[ccp->cmd_q_count];
814 ccp->cmd_q_count++;
815
816 cmd_q->ccp = ccp;
817 cmd_q->id = i;
818 cmd_q->dma_pool = dma_pool;
819 mutex_init(&cmd_q->q_mutex);
820
821 /* Page alignment satisfies our needs for N <= 128 */
822 BUILD_BUG_ON(COMMANDS_PER_QUEUE > 128);
823 cmd_q->qsize = Q_SIZE(Q_DESC_SIZE);
824 cmd_q->qbase = dma_zalloc_coherent(dev, cmd_q->qsize,
825 &cmd_q->qbase_dma,
826 GFP_KERNEL);
827 if (!cmd_q->qbase) {
828 dev_err(dev, "unable to allocate command queue\n");
829 ret = -ENOMEM;
830 goto e_pool;
831 }
832
833 cmd_q->qidx = 0;
834 /* Preset some register values and masks that are queue
835 * number dependent
836 */
837 cmd_q->reg_control = ccp->io_regs +
838 CMD5_Q_STATUS_INCR * (i + 1);
839 cmd_q->reg_tail_lo = cmd_q->reg_control + CMD5_Q_TAIL_LO_BASE;
840 cmd_q->reg_head_lo = cmd_q->reg_control + CMD5_Q_HEAD_LO_BASE;
841 cmd_q->reg_int_enable = cmd_q->reg_control +
842 CMD5_Q_INT_ENABLE_BASE;
843 cmd_q->reg_interrupt_status = cmd_q->reg_control +
844 CMD5_Q_INTERRUPT_STATUS_BASE;
845 cmd_q->reg_status = cmd_q->reg_control + CMD5_Q_STATUS_BASE;
846 cmd_q->reg_int_status = cmd_q->reg_control +
847 CMD5_Q_INT_STATUS_BASE;
848 cmd_q->reg_dma_status = cmd_q->reg_control +
849 CMD5_Q_DMA_STATUS_BASE;
850 cmd_q->reg_dma_read_status = cmd_q->reg_control +
851 CMD5_Q_DMA_READ_STATUS_BASE;
852 cmd_q->reg_dma_write_status = cmd_q->reg_control +
853 CMD5_Q_DMA_WRITE_STATUS_BASE;
854
855 init_waitqueue_head(&cmd_q->int_queue);
856
857 dev_dbg(dev, "queue #%u available\n", i);
858 }
Gary R Hook990672d2017-03-15 13:20:52 -0500859
Gary R Hook4b394a22016-07-26 19:10:21 -0500860 if (ccp->cmd_q_count == 0) {
861 dev_notice(dev, "no command queues available\n");
862 ret = -EIO;
863 goto e_pool;
864 }
Gary R Hook4b394a22016-07-26 19:10:21 -0500865
866 /* Turn off the queues and disable interrupts until ready */
Gary R Hook6263b512017-04-21 10:50:14 -0500867 ccp5_disable_queue_interrupts(ccp);
Gary R Hook4b394a22016-07-26 19:10:21 -0500868 for (i = 0; i < ccp->cmd_q_count; i++) {
869 cmd_q = &ccp->cmd_q[i];
870
871 cmd_q->qcontrol = 0; /* Start with nothing */
872 iowrite32(cmd_q->qcontrol, cmd_q->reg_control);
873
Gary R Hook4b394a22016-07-26 19:10:21 -0500874 ioread32(cmd_q->reg_int_status);
875 ioread32(cmd_q->reg_status);
876
Gary R Hook6263b512017-04-21 10:50:14 -0500877 /* Clear the interrupt status */
Gary R Hook56467cb2017-04-20 15:24:09 -0500878 iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_interrupt_status);
Gary R Hook4b394a22016-07-26 19:10:21 -0500879 }
880
881 dev_dbg(dev, "Requesting an IRQ...\n");
882 /* Request an irq */
Brijesh Singhf4d18d62017-07-06 09:59:15 -0500883 ret = sp_request_ccp_irq(ccp->sp, ccp5_irq_handler, ccp->name, ccp);
Gary R Hook4b394a22016-07-26 19:10:21 -0500884 if (ret) {
885 dev_err(dev, "unable to allocate an IRQ\n");
886 goto e_pool;
887 }
Gary R Hook6263b512017-04-21 10:50:14 -0500888 /* Initialize the ISR tasklet */
889 if (ccp->use_tasklet)
890 tasklet_init(&ccp->irq_tasklet, ccp5_irq_bh,
891 (unsigned long)ccp);
Gary R Hook4b394a22016-07-26 19:10:21 -0500892
Gary R Hook4b394a22016-07-26 19:10:21 -0500893 dev_dbg(dev, "Loading LSB map...\n");
894 /* Copy the private LSB mask to the public registers */
895 status_lo = ioread32(ccp->io_regs + LSB_PRIVATE_MASK_LO_OFFSET);
896 status_hi = ioread32(ccp->io_regs + LSB_PRIVATE_MASK_HI_OFFSET);
897 iowrite32(status_lo, ccp->io_regs + LSB_PUBLIC_MASK_LO_OFFSET);
898 iowrite32(status_hi, ccp->io_regs + LSB_PUBLIC_MASK_HI_OFFSET);
899 status = ((u64)status_hi<<30) | (u64)status_lo;
900
901 dev_dbg(dev, "Configuring virtual queues...\n");
902 /* Configure size of each virtual queue accessible to host */
903 for (i = 0; i < ccp->cmd_q_count; i++) {
904 u32 dma_addr_lo;
905 u32 dma_addr_hi;
906
907 cmd_q = &ccp->cmd_q[i];
908
909 cmd_q->qcontrol &= ~(CMD5_Q_SIZE << CMD5_Q_SHIFT);
910 cmd_q->qcontrol |= QUEUE_SIZE_VAL << CMD5_Q_SHIFT;
911
912 cmd_q->qdma_tail = cmd_q->qbase_dma;
913 dma_addr_lo = low_address(cmd_q->qdma_tail);
914 iowrite32((u32)dma_addr_lo, cmd_q->reg_tail_lo);
915 iowrite32((u32)dma_addr_lo, cmd_q->reg_head_lo);
916
917 dma_addr_hi = high_address(cmd_q->qdma_tail);
918 cmd_q->qcontrol |= (dma_addr_hi << 16);
919 iowrite32(cmd_q->qcontrol, cmd_q->reg_control);
920
921 /* Find the LSB regions accessible to the queue */
922 ccp_find_lsb_regions(cmd_q, status);
923 cmd_q->lsb = -1; /* Unassigned value */
924 }
925
926 dev_dbg(dev, "Assigning LSBs...\n");
927 ret = ccp_assign_lsbs(ccp);
928 if (ret) {
929 dev_err(dev, "Unable to assign LSBs (%d)\n", ret);
930 goto e_irq;
931 }
932
933 /* Optimization: pre-allocate LSB slots for each queue */
934 for (i = 0; i < ccp->cmd_q_count; i++) {
935 ccp->cmd_q[i].sb_key = ccp_lsb_alloc(&ccp->cmd_q[i], 2);
936 ccp->cmd_q[i].sb_ctx = ccp_lsb_alloc(&ccp->cmd_q[i], 2);
937 }
938
939 dev_dbg(dev, "Starting threads...\n");
940 /* Create a kthread for each queue */
941 for (i = 0; i < ccp->cmd_q_count; i++) {
942 struct task_struct *kthread;
943
944 cmd_q = &ccp->cmd_q[i];
945
946 kthread = kthread_create(ccp_cmd_queue_thread, cmd_q,
947 "%s-q%u", ccp->name, cmd_q->id);
948 if (IS_ERR(kthread)) {
949 dev_err(dev, "error creating queue thread (%ld)\n",
950 PTR_ERR(kthread));
951 ret = PTR_ERR(kthread);
952 goto e_kthread;
953 }
954
955 cmd_q->kthread = kthread;
956 wake_up_process(kthread);
957 }
958
959 dev_dbg(dev, "Enabling interrupts...\n");
Gary R Hook6263b512017-04-21 10:50:14 -0500960 ccp5_enable_queue_interrupts(ccp);
Gary R Hook4b394a22016-07-26 19:10:21 -0500961
962 dev_dbg(dev, "Registering device...\n");
963 /* Put this on the unit list to make it available */
964 ccp_add_device(ccp);
965
Gary R Hook084935b2016-07-26 19:10:31 -0500966 ret = ccp_register_rng(ccp);
967 if (ret)
968 goto e_kthread;
969
Gary R Hook99d90b22016-07-26 19:10:40 -0500970 /* Register the DMA engine support */
971 ret = ccp_dmaengine_register(ccp);
972 if (ret)
Gary R Hook9ddb9dc2016-09-28 11:53:47 -0500973 goto e_hwrng;
Gary R Hook99d90b22016-07-26 19:10:40 -0500974
Gary R Hook3cdbe342017-05-02 17:33:40 -0500975 /* Set up debugfs entries */
976 ccp5_debugfs_setup(ccp);
977
Gary R Hook4b394a22016-07-26 19:10:21 -0500978 return 0;
979
Gary R Hook9ddb9dc2016-09-28 11:53:47 -0500980e_hwrng:
981 ccp_unregister_rng(ccp);
982
Gary R Hook4b394a22016-07-26 19:10:21 -0500983e_kthread:
984 for (i = 0; i < ccp->cmd_q_count; i++)
985 if (ccp->cmd_q[i].kthread)
986 kthread_stop(ccp->cmd_q[i].kthread);
987
988e_irq:
Brijesh Singhf4d18d62017-07-06 09:59:15 -0500989 sp_free_ccp_irq(ccp->sp, ccp);
Gary R Hook4b394a22016-07-26 19:10:21 -0500990
991e_pool:
992 for (i = 0; i < ccp->cmd_q_count; i++)
993 dma_pool_destroy(ccp->cmd_q[i].dma_pool);
994
995 return ret;
996}
997
998static void ccp5_destroy(struct ccp_device *ccp)
999{
1000 struct device *dev = ccp->dev;
1001 struct ccp_cmd_queue *cmd_q;
1002 struct ccp_cmd *cmd;
1003 unsigned int i;
1004
Gary R Hook99d90b22016-07-26 19:10:40 -05001005 /* Unregister the DMA engine */
1006 ccp_dmaengine_unregister(ccp);
1007
Gary R Hook084935b2016-07-26 19:10:31 -05001008 /* Unregister the RNG */
1009 ccp_unregister_rng(ccp);
1010
Gary R Hook4b394a22016-07-26 19:10:21 -05001011 /* Remove this device from the list of available units first */
1012 ccp_del_device(ccp);
1013
Gary R Hook3cdbe342017-05-02 17:33:40 -05001014 /* We're in the process of tearing down the entire driver;
1015 * when all the devices are gone clean up debugfs
1016 */
1017 if (ccp_present())
1018 ccp5_debugfs_destroy();
1019
Gary R Hook4b394a22016-07-26 19:10:21 -05001020 /* Disable and clear interrupts */
Gary R Hook6263b512017-04-21 10:50:14 -05001021 ccp5_disable_queue_interrupts(ccp);
Gary R Hook4b394a22016-07-26 19:10:21 -05001022 for (i = 0; i < ccp->cmd_q_count; i++) {
1023 cmd_q = &ccp->cmd_q[i];
1024
1025 /* Turn off the run bit */
1026 iowrite32(cmd_q->qcontrol & ~CMD5_Q_RUN, cmd_q->reg_control);
1027
Gary R Hook4b394a22016-07-26 19:10:21 -05001028 /* Clear the interrupt status */
Gary R Hook116591f2017-04-20 15:24:22 -05001029 iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_interrupt_status);
Gary R Hook4b394a22016-07-26 19:10:21 -05001030 ioread32(cmd_q->reg_int_status);
1031 ioread32(cmd_q->reg_status);
1032 }
1033
1034 /* Stop the queue kthreads */
1035 for (i = 0; i < ccp->cmd_q_count; i++)
1036 if (ccp->cmd_q[i].kthread)
1037 kthread_stop(ccp->cmd_q[i].kthread);
1038
Brijesh Singhf4d18d62017-07-06 09:59:15 -05001039 sp_free_ccp_irq(ccp->sp, ccp);
Gary R Hook4b394a22016-07-26 19:10:21 -05001040
1041 for (i = 0; i < ccp->cmd_q_count; i++) {
1042 cmd_q = &ccp->cmd_q[i];
1043 dma_free_coherent(dev, cmd_q->qsize, cmd_q->qbase,
1044 cmd_q->qbase_dma);
1045 }
1046
1047 /* Flush the cmd and backlog queue */
1048 while (!list_empty(&ccp->cmd)) {
1049 /* Invoke the callback directly with an error code */
1050 cmd = list_first_entry(&ccp->cmd, struct ccp_cmd, entry);
1051 list_del(&cmd->entry);
1052 cmd->callback(cmd->data, -ENODEV);
1053 }
1054 while (!list_empty(&ccp->backlog)) {
1055 /* Invoke the callback directly with an error code */
1056 cmd = list_first_entry(&ccp->backlog, struct ccp_cmd, entry);
1057 list_del(&cmd->entry);
1058 cmd->callback(cmd->data, -ENODEV);
1059 }
1060}
1061
Gary R Hook4b394a22016-07-26 19:10:21 -05001062static void ccp5_config(struct ccp_device *ccp)
1063{
1064 /* Public side */
Gary R Hook500c0102017-01-27 15:28:45 -06001065 iowrite32(0x0, ccp->io_regs + CMD5_REQID_CONFIG_OFFSET);
Gary R Hook4b394a22016-07-26 19:10:21 -05001066}
1067
Gary R Hooke14e7d12016-07-26 19:10:49 -05001068static void ccp5other_config(struct ccp_device *ccp)
1069{
1070 int i;
1071 u32 rnd;
1072
1073 /* We own all of the queues on the NTB CCP */
1074
1075 iowrite32(0x00012D57, ccp->io_regs + CMD5_TRNG_CTL_OFFSET);
1076 iowrite32(0x00000003, ccp->io_regs + CMD5_CONFIG_0_OFFSET);
1077 for (i = 0; i < 12; i++) {
1078 rnd = ioread32(ccp->io_regs + TRNG_OUT_REG);
1079 iowrite32(rnd, ccp->io_regs + CMD5_AES_MASK_OFFSET);
1080 }
1081
1082 iowrite32(0x0000001F, ccp->io_regs + CMD5_QUEUE_MASK_OFFSET);
1083 iowrite32(0x00005B6D, ccp->io_regs + CMD5_QUEUE_PRIO_OFFSET);
1084 iowrite32(0x00000000, ccp->io_regs + CMD5_CMD_TIMEOUT_OFFSET);
1085
1086 iowrite32(0x3FFFFFFF, ccp->io_regs + LSB_PRIVATE_MASK_LO_OFFSET);
1087 iowrite32(0x000003FF, ccp->io_regs + LSB_PRIVATE_MASK_HI_OFFSET);
1088
1089 iowrite32(0x00108823, ccp->io_regs + CMD5_CLK_GATE_CTL_OFFSET);
1090
1091 ccp5_config(ccp);
1092}
1093
1094/* Version 5 adds some function, but is essentially the same as v5 */
Gary R Hook4b394a22016-07-26 19:10:21 -05001095static const struct ccp_actions ccp5_actions = {
1096 .aes = ccp5_perform_aes,
1097 .xts_aes = ccp5_perform_xts_aes,
1098 .sha = ccp5_perform_sha,
Gary R Hook990672d2017-03-15 13:20:52 -05001099 .des3 = ccp5_perform_des3,
Gary R Hook4b394a22016-07-26 19:10:21 -05001100 .rsa = ccp5_perform_rsa,
1101 .passthru = ccp5_perform_passthru,
1102 .ecc = ccp5_perform_ecc,
1103 .sballoc = ccp_lsb_alloc,
1104 .sbfree = ccp_lsb_free,
1105 .init = ccp5_init,
1106 .destroy = ccp5_destroy,
1107 .get_free_slots = ccp5_get_free_slots,
Gary R Hook4b394a22016-07-26 19:10:21 -05001108};
1109
Gary R Hook9ddb9dc2016-09-28 11:53:47 -05001110const struct ccp_vdata ccpv5a = {
Gary R Hook4b394a22016-07-26 19:10:21 -05001111 .version = CCP_VERSION(5, 0),
1112 .setup = ccp5_config,
1113 .perform = &ccp5_actions,
Gary R Hook4b394a22016-07-26 19:10:21 -05001114 .offset = 0x0,
Gary R Hooke28c1902017-07-17 15:16:42 -05001115 .rsamax = CCP5_RSA_MAX_WIDTH,
Gary R Hook4b394a22016-07-26 19:10:21 -05001116};
Gary R Hooke14e7d12016-07-26 19:10:49 -05001117
Gary R Hook9ddb9dc2016-09-28 11:53:47 -05001118const struct ccp_vdata ccpv5b = {
Gary R Hooke14e7d12016-07-26 19:10:49 -05001119 .version = CCP_VERSION(5, 0),
Gary R Hookefc989f2017-03-23 12:53:30 -05001120 .dma_chan_attr = DMA_PRIVATE,
Gary R Hooke14e7d12016-07-26 19:10:49 -05001121 .setup = ccp5other_config,
1122 .perform = &ccp5_actions,
Gary R Hooke14e7d12016-07-26 19:10:49 -05001123 .offset = 0x0,
Gary R Hooke28c1902017-07-17 15:16:42 -05001124 .rsamax = CCP5_RSA_MAX_WIDTH,
Gary R Hooke14e7d12016-07-26 19:10:49 -05001125};