2
* 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru>
5
* This program is free software; you can redistribute it and/or modify
6
* it under the terms of the GNU General Public License as published by
7
* the Free Software Foundation; either version 2 of the License, or
8
* (at your option) any later version.
10
* This program is distributed in the hope that it will be useful,
11
* but WITHOUT ANY WARRANTY; without even the implied warranty of
12
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
* GNU General Public License for more details.
15
* You should have received a copy of the GNU General Public License
16
* along with this program; if not, write to the Free Software
17
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20
#include <linux/kernel.h>
21
#include <linux/module.h>
22
#include <linux/moduleparam.h>
23
#include <linux/mod_devicetable.h>
24
#include <linux/interrupt.h>
25
#include <linux/pci.h>
26
#include <linux/slab.h>
27
#include <linux/delay.h>
29
#include <linux/dma-mapping.h>
30
#include <linux/scatterlist.h>
31
#include <linux/highmem.h>
32
#include <linux/crypto.h>
33
#include <linux/hw_random.h>
34
#include <linux/ktime.h>
36
#include <crypto/algapi.h>
37
#include <crypto/des.h>
39
#include <asm/kmap_types.h>
44
#define dprintk(f, a...) printk(f, ##a)
46
#define dprintk(f, a...) do {} while (0)
49
static char hifn_pll_ref[sizeof("extNNN")] = "ext";
50
module_param_string(hifn_pll_ref, hifn_pll_ref, sizeof(hifn_pll_ref), 0444);
51
MODULE_PARM_DESC(hifn_pll_ref,
52
"PLL reference clock (pci[freq] or ext[freq], default ext)");
54
static atomic_t hifn_dev_number;
56
#define ACRYPTO_OP_DECRYPT 0
57
#define ACRYPTO_OP_ENCRYPT 1
58
#define ACRYPTO_OP_HMAC 2
59
#define ACRYPTO_OP_RNG 3
61
#define ACRYPTO_MODE_ECB 0
62
#define ACRYPTO_MODE_CBC 1
63
#define ACRYPTO_MODE_CFB 2
64
#define ACRYPTO_MODE_OFB 3
66
#define ACRYPTO_TYPE_AES_128 0
67
#define ACRYPTO_TYPE_AES_192 1
68
#define ACRYPTO_TYPE_AES_256 2
69
#define ACRYPTO_TYPE_3DES 3
70
#define ACRYPTO_TYPE_DES 4
72
#define PCI_VENDOR_ID_HIFN 0x13A3
73
#define PCI_DEVICE_ID_HIFN_7955 0x0020
74
#define PCI_DEVICE_ID_HIFN_7956 0x001d
76
/* I/O region sizes */
78
#define HIFN_BAR0_SIZE 0x1000
79
#define HIFN_BAR1_SIZE 0x2000
80
#define HIFN_BAR2_SIZE 0x8000
84
#define HIFN_DMA_CRA 0x0C /* DMA Command Ring Address */
85
#define HIFN_DMA_SDRA 0x1C /* DMA Source Data Ring Address */
86
#define HIFN_DMA_RRA 0x2C /* DMA Result Ring Address */
87
#define HIFN_DMA_DDRA 0x3C /* DMA Destination Data Ring Address */
88
#define HIFN_DMA_STCTL 0x40 /* DMA Status and Control */
89
#define HIFN_DMA_INTREN 0x44 /* DMA Interrupt Enable */
90
#define HIFN_DMA_CFG1 0x48 /* DMA Configuration #1 */
91
#define HIFN_DMA_CFG2 0x6C /* DMA Configuration #2 */
92
#define HIFN_CHIP_ID 0x98 /* Chip ID */
95
* Processing Unit Registers (offset from BASEREG0)
97
#define HIFN_0_PUDATA 0x00 /* Processing Unit Data */
98
#define HIFN_0_PUCTRL 0x04 /* Processing Unit Control */
99
#define HIFN_0_PUISR 0x08 /* Processing Unit Interrupt Status */
100
#define HIFN_0_PUCNFG 0x0c /* Processing Unit Configuration */
101
#define HIFN_0_PUIER 0x10 /* Processing Unit Interrupt Enable */
102
#define HIFN_0_PUSTAT 0x14 /* Processing Unit Status/Chip ID */
103
#define HIFN_0_FIFOSTAT 0x18 /* FIFO Status */
104
#define HIFN_0_FIFOCNFG 0x1c /* FIFO Configuration */
105
#define HIFN_0_SPACESIZE 0x20 /* Register space size */
107
/* Processing Unit Control Register (HIFN_0_PUCTRL) */
108
#define HIFN_PUCTRL_CLRSRCFIFO 0x0010 /* clear source fifo */
109
#define HIFN_PUCTRL_STOP 0x0008 /* stop pu */
110
#define HIFN_PUCTRL_LOCKRAM 0x0004 /* lock ram */
111
#define HIFN_PUCTRL_DMAENA 0x0002 /* enable dma */
112
#define HIFN_PUCTRL_RESET 0x0001 /* Reset processing unit */
114
/* Processing Unit Interrupt Status Register (HIFN_0_PUISR) */
115
#define HIFN_PUISR_CMDINVAL 0x8000 /* Invalid command interrupt */
116
#define HIFN_PUISR_DATAERR 0x4000 /* Data error interrupt */
117
#define HIFN_PUISR_SRCFIFO 0x2000 /* Source FIFO ready interrupt */
118
#define HIFN_PUISR_DSTFIFO 0x1000 /* Destination FIFO ready interrupt */
119
#define HIFN_PUISR_DSTOVER 0x0200 /* Destination overrun interrupt */
120
#define HIFN_PUISR_SRCCMD 0x0080 /* Source command interrupt */
121
#define HIFN_PUISR_SRCCTX 0x0040 /* Source context interrupt */
122
#define HIFN_PUISR_SRCDATA 0x0020 /* Source data interrupt */
123
#define HIFN_PUISR_DSTDATA 0x0010 /* Destination data interrupt */
124
#define HIFN_PUISR_DSTRESULT 0x0004 /* Destination result interrupt */
126
/* Processing Unit Configuration Register (HIFN_0_PUCNFG) */
127
#define HIFN_PUCNFG_DRAMMASK 0xe000 /* DRAM size mask */
128
#define HIFN_PUCNFG_DSZ_256K 0x0000 /* 256k dram */
129
#define HIFN_PUCNFG_DSZ_512K 0x2000 /* 512k dram */
130
#define HIFN_PUCNFG_DSZ_1M 0x4000 /* 1m dram */
131
#define HIFN_PUCNFG_DSZ_2M 0x6000 /* 2m dram */
132
#define HIFN_PUCNFG_DSZ_4M 0x8000 /* 4m dram */
133
#define HIFN_PUCNFG_DSZ_8M 0xa000 /* 8m dram */
134
#define HIFN_PUNCFG_DSZ_16M 0xc000 /* 16m dram */
135
#define HIFN_PUCNFG_DSZ_32M 0xe000 /* 32m dram */
136
#define HIFN_PUCNFG_DRAMREFRESH 0x1800 /* DRAM refresh rate mask */
137
#define HIFN_PUCNFG_DRFR_512 0x0000 /* 512 divisor of ECLK */
138
#define HIFN_PUCNFG_DRFR_256 0x0800 /* 256 divisor of ECLK */
139
#define HIFN_PUCNFG_DRFR_128 0x1000 /* 128 divisor of ECLK */
140
#define HIFN_PUCNFG_TCALLPHASES 0x0200 /* your guess is as good as mine... */
141
#define HIFN_PUCNFG_TCDRVTOTEM 0x0100 /* your guess is as good as mine... */
142
#define HIFN_PUCNFG_BIGENDIAN 0x0080 /* DMA big endian mode */
143
#define HIFN_PUCNFG_BUS32 0x0040 /* Bus width 32bits */
144
#define HIFN_PUCNFG_BUS16 0x0000 /* Bus width 16 bits */
145
#define HIFN_PUCNFG_CHIPID 0x0020 /* Allow chipid from PUSTAT */
146
#define HIFN_PUCNFG_DRAM 0x0010 /* Context RAM is DRAM */
147
#define HIFN_PUCNFG_SRAM 0x0000 /* Context RAM is SRAM */
148
#define HIFN_PUCNFG_COMPSING 0x0004 /* Enable single compression context */
149
#define HIFN_PUCNFG_ENCCNFG 0x0002 /* Encryption configuration */
151
/* Processing Unit Interrupt Enable Register (HIFN_0_PUIER) */
152
#define HIFN_PUIER_CMDINVAL 0x8000 /* Invalid command interrupt */
153
#define HIFN_PUIER_DATAERR 0x4000 /* Data error interrupt */
154
#define HIFN_PUIER_SRCFIFO 0x2000 /* Source FIFO ready interrupt */
155
#define HIFN_PUIER_DSTFIFO 0x1000 /* Destination FIFO ready interrupt */
156
#define HIFN_PUIER_DSTOVER 0x0200 /* Destination overrun interrupt */
157
#define HIFN_PUIER_SRCCMD 0x0080 /* Source command interrupt */
158
#define HIFN_PUIER_SRCCTX 0x0040 /* Source context interrupt */
159
#define HIFN_PUIER_SRCDATA 0x0020 /* Source data interrupt */
160
#define HIFN_PUIER_DSTDATA 0x0010 /* Destination data interrupt */
161
#define HIFN_PUIER_DSTRESULT 0x0004 /* Destination result interrupt */
163
/* Processing Unit Status Register/Chip ID (HIFN_0_PUSTAT) */
164
#define HIFN_PUSTAT_CMDINVAL 0x8000 /* Invalid command interrupt */
165
#define HIFN_PUSTAT_DATAERR 0x4000 /* Data error interrupt */
166
#define HIFN_PUSTAT_SRCFIFO 0x2000 /* Source FIFO ready interrupt */
167
#define HIFN_PUSTAT_DSTFIFO 0x1000 /* Destination FIFO ready interrupt */
168
#define HIFN_PUSTAT_DSTOVER 0x0200 /* Destination overrun interrupt */
169
#define HIFN_PUSTAT_SRCCMD 0x0080 /* Source command interrupt */
170
#define HIFN_PUSTAT_SRCCTX 0x0040 /* Source context interrupt */
171
#define HIFN_PUSTAT_SRCDATA 0x0020 /* Source data interrupt */
172
#define HIFN_PUSTAT_DSTDATA 0x0010 /* Destination data interrupt */
173
#define HIFN_PUSTAT_DSTRESULT 0x0004 /* Destination result interrupt */
174
#define HIFN_PUSTAT_CHIPREV 0x00ff /* Chip revision mask */
175
#define HIFN_PUSTAT_CHIPENA 0xff00 /* Chip enabled mask */
176
#define HIFN_PUSTAT_ENA_2 0x1100 /* Level 2 enabled */
177
#define HIFN_PUSTAT_ENA_1 0x1000 /* Level 1 enabled */
178
#define HIFN_PUSTAT_ENA_0 0x3000 /* Level 0 enabled */
179
#define HIFN_PUSTAT_REV_2 0x0020 /* 7751 PT6/2 */
180
#define HIFN_PUSTAT_REV_3 0x0030 /* 7751 PT6/3 */
182
/* FIFO Status Register (HIFN_0_FIFOSTAT) */
183
#define HIFN_FIFOSTAT_SRC 0x7f00 /* Source FIFO available */
184
#define HIFN_FIFOSTAT_DST 0x007f /* Destination FIFO available */
186
/* FIFO Configuration Register (HIFN_0_FIFOCNFG) */
187
#define HIFN_FIFOCNFG_THRESHOLD 0x0400 /* must be written as 1 */
190
* DMA Interface Registers (offset from BASEREG1)
192
#define HIFN_1_DMA_CRAR 0x0c /* DMA Command Ring Address */
193
#define HIFN_1_DMA_SRAR 0x1c /* DMA Source Ring Address */
194
#define HIFN_1_DMA_RRAR 0x2c /* DMA Result Ring Address */
195
#define HIFN_1_DMA_DRAR 0x3c /* DMA Destination Ring Address */
196
#define HIFN_1_DMA_CSR 0x40 /* DMA Status and Control */
197
#define HIFN_1_DMA_IER 0x44 /* DMA Interrupt Enable */
198
#define HIFN_1_DMA_CNFG 0x48 /* DMA Configuration */
199
#define HIFN_1_PLL 0x4c /* 795x: PLL config */
200
#define HIFN_1_7811_RNGENA 0x60 /* 7811: rng enable */
201
#define HIFN_1_7811_RNGCFG 0x64 /* 7811: rng config */
202
#define HIFN_1_7811_RNGDAT 0x68 /* 7811: rng data */
203
#define HIFN_1_7811_RNGSTS 0x6c /* 7811: rng status */
204
#define HIFN_1_7811_MIPSRST 0x94 /* 7811: MIPS reset */
205
#define HIFN_1_REVID 0x98 /* Revision ID */
206
#define HIFN_1_UNLOCK_SECRET1 0xf4
207
#define HIFN_1_UNLOCK_SECRET2 0xfc
208
#define HIFN_1_PUB_RESET 0x204 /* Public/RNG Reset */
209
#define HIFN_1_PUB_BASE 0x300 /* Public Base Address */
210
#define HIFN_1_PUB_OPLEN 0x304 /* Public Operand Length */
211
#define HIFN_1_PUB_OP 0x308 /* Public Operand */
212
#define HIFN_1_PUB_STATUS 0x30c /* Public Status */
213
#define HIFN_1_PUB_IEN 0x310 /* Public Interrupt enable */
214
#define HIFN_1_RNG_CONFIG 0x314 /* RNG config */
215
#define HIFN_1_RNG_DATA 0x318 /* RNG data */
216
#define HIFN_1_PUB_MEM 0x400 /* start of Public key memory */
217
#define HIFN_1_PUB_MEMEND 0xbff /* end of Public key memory */
219
/* DMA Status and Control Register (HIFN_1_DMA_CSR) */
220
#define HIFN_DMACSR_D_CTRLMASK 0xc0000000 /* Destinition Ring Control */
221
#define HIFN_DMACSR_D_CTRL_NOP 0x00000000 /* Dest. Control: no-op */
222
#define HIFN_DMACSR_D_CTRL_DIS 0x40000000 /* Dest. Control: disable */
223
#define HIFN_DMACSR_D_CTRL_ENA 0x80000000 /* Dest. Control: enable */
224
#define HIFN_DMACSR_D_ABORT 0x20000000 /* Destinition Ring PCIAbort */
225
#define HIFN_DMACSR_D_DONE 0x10000000 /* Destinition Ring Done */
226
#define HIFN_DMACSR_D_LAST 0x08000000 /* Destinition Ring Last */
227
#define HIFN_DMACSR_D_WAIT 0x04000000 /* Destinition Ring Waiting */
228
#define HIFN_DMACSR_D_OVER 0x02000000 /* Destinition Ring Overflow */
229
#define HIFN_DMACSR_R_CTRL 0x00c00000 /* Result Ring Control */
230
#define HIFN_DMACSR_R_CTRL_NOP 0x00000000 /* Result Control: no-op */
231
#define HIFN_DMACSR_R_CTRL_DIS 0x00400000 /* Result Control: disable */
232
#define HIFN_DMACSR_R_CTRL_ENA 0x00800000 /* Result Control: enable */
233
#define HIFN_DMACSR_R_ABORT 0x00200000 /* Result Ring PCI Abort */
234
#define HIFN_DMACSR_R_DONE 0x00100000 /* Result Ring Done */
235
#define HIFN_DMACSR_R_LAST 0x00080000 /* Result Ring Last */
236
#define HIFN_DMACSR_R_WAIT 0x00040000 /* Result Ring Waiting */
237
#define HIFN_DMACSR_R_OVER 0x00020000 /* Result Ring Overflow */
238
#define HIFN_DMACSR_S_CTRL 0x0000c000 /* Source Ring Control */
239
#define HIFN_DMACSR_S_CTRL_NOP 0x00000000 /* Source Control: no-op */
240
#define HIFN_DMACSR_S_CTRL_DIS 0x00004000 /* Source Control: disable */
241
#define HIFN_DMACSR_S_CTRL_ENA 0x00008000 /* Source Control: enable */
242
#define HIFN_DMACSR_S_ABORT 0x00002000 /* Source Ring PCI Abort */
243
#define HIFN_DMACSR_S_DONE 0x00001000 /* Source Ring Done */
244
#define HIFN_DMACSR_S_LAST 0x00000800 /* Source Ring Last */
245
#define HIFN_DMACSR_S_WAIT 0x00000400 /* Source Ring Waiting */
246
#define HIFN_DMACSR_ILLW 0x00000200 /* Illegal write (7811 only) */
247
#define HIFN_DMACSR_ILLR 0x00000100 /* Illegal read (7811 only) */
248
#define HIFN_DMACSR_C_CTRL 0x000000c0 /* Command Ring Control */
249
#define HIFN_DMACSR_C_CTRL_NOP 0x00000000 /* Command Control: no-op */
250
#define HIFN_DMACSR_C_CTRL_DIS 0x00000040 /* Command Control: disable */
251
#define HIFN_DMACSR_C_CTRL_ENA 0x00000080 /* Command Control: enable */
252
#define HIFN_DMACSR_C_ABORT 0x00000020 /* Command Ring PCI Abort */
253
#define HIFN_DMACSR_C_DONE 0x00000010 /* Command Ring Done */
254
#define HIFN_DMACSR_C_LAST 0x00000008 /* Command Ring Last */
255
#define HIFN_DMACSR_C_WAIT 0x00000004 /* Command Ring Waiting */
256
#define HIFN_DMACSR_PUBDONE 0x00000002 /* Public op done (7951 only) */
257
#define HIFN_DMACSR_ENGINE 0x00000001 /* Command Ring Engine IRQ */
259
/* DMA Interrupt Enable Register (HIFN_1_DMA_IER) */
260
#define HIFN_DMAIER_D_ABORT 0x20000000 /* Destination Ring PCIAbort */
261
#define HIFN_DMAIER_D_DONE 0x10000000 /* Destination Ring Done */
262
#define HIFN_DMAIER_D_LAST 0x08000000 /* Destination Ring Last */
263
#define HIFN_DMAIER_D_WAIT 0x04000000 /* Destination Ring Waiting */
264
#define HIFN_DMAIER_D_OVER 0x02000000 /* Destination Ring Overflow */
265
#define HIFN_DMAIER_R_ABORT 0x00200000 /* Result Ring PCI Abort */
266
#define HIFN_DMAIER_R_DONE 0x00100000 /* Result Ring Done */
267
#define HIFN_DMAIER_R_LAST 0x00080000 /* Result Ring Last */
268
#define HIFN_DMAIER_R_WAIT 0x00040000 /* Result Ring Waiting */
269
#define HIFN_DMAIER_R_OVER 0x00020000 /* Result Ring Overflow */
270
#define HIFN_DMAIER_S_ABORT 0x00002000 /* Source Ring PCI Abort */
271
#define HIFN_DMAIER_S_DONE 0x00001000 /* Source Ring Done */
272
#define HIFN_DMAIER_S_LAST 0x00000800 /* Source Ring Last */
273
#define HIFN_DMAIER_S_WAIT 0x00000400 /* Source Ring Waiting */
274
#define HIFN_DMAIER_ILLW 0x00000200 /* Illegal write (7811 only) */
275
#define HIFN_DMAIER_ILLR 0x00000100 /* Illegal read (7811 only) */
276
#define HIFN_DMAIER_C_ABORT 0x00000020 /* Command Ring PCI Abort */
277
#define HIFN_DMAIER_C_DONE 0x00000010 /* Command Ring Done */
278
#define HIFN_DMAIER_C_LAST 0x00000008 /* Command Ring Last */
279
#define HIFN_DMAIER_C_WAIT 0x00000004 /* Command Ring Waiting */
280
#define HIFN_DMAIER_PUBDONE 0x00000002 /* public op done (7951 only) */
281
#define HIFN_DMAIER_ENGINE 0x00000001 /* Engine IRQ */
283
/* DMA Configuration Register (HIFN_1_DMA_CNFG) */
284
#define HIFN_DMACNFG_BIGENDIAN 0x10000000 /* big endian mode */
285
#define HIFN_DMACNFG_POLLFREQ 0x00ff0000 /* Poll frequency mask */
286
#define HIFN_DMACNFG_UNLOCK 0x00000800
287
#define HIFN_DMACNFG_POLLINVAL 0x00000700 /* Invalid Poll Scalar */
288
#define HIFN_DMACNFG_LAST 0x00000010 /* Host control LAST bit */
289
#define HIFN_DMACNFG_MODE 0x00000004 /* DMA mode */
290
#define HIFN_DMACNFG_DMARESET 0x00000002 /* DMA Reset # */
291
#define HIFN_DMACNFG_MSTRESET 0x00000001 /* Master Reset # */
293
/* PLL configuration register */
294
#define HIFN_PLL_REF_CLK_HBI 0x00000000 /* HBI reference clock */
295
#define HIFN_PLL_REF_CLK_PLL 0x00000001 /* PLL reference clock */
296
#define HIFN_PLL_BP 0x00000002 /* Reference clock bypass */
297
#define HIFN_PLL_PK_CLK_HBI 0x00000000 /* PK engine HBI clock */
298
#define HIFN_PLL_PK_CLK_PLL 0x00000008 /* PK engine PLL clock */
299
#define HIFN_PLL_PE_CLK_HBI 0x00000000 /* PE engine HBI clock */
300
#define HIFN_PLL_PE_CLK_PLL 0x00000010 /* PE engine PLL clock */
301
#define HIFN_PLL_RESERVED_1 0x00000400 /* Reserved bit, must be 1 */
302
#define HIFN_PLL_ND_SHIFT 11 /* Clock multiplier shift */
303
#define HIFN_PLL_ND_MULT_2 0x00000000 /* PLL clock multiplier 2 */
304
#define HIFN_PLL_ND_MULT_4 0x00000800 /* PLL clock multiplier 4 */
305
#define HIFN_PLL_ND_MULT_6 0x00001000 /* PLL clock multiplier 6 */
306
#define HIFN_PLL_ND_MULT_8 0x00001800 /* PLL clock multiplier 8 */
307
#define HIFN_PLL_ND_MULT_10 0x00002000 /* PLL clock multiplier 10 */
308
#define HIFN_PLL_ND_MULT_12 0x00002800 /* PLL clock multiplier 12 */
309
#define HIFN_PLL_IS_1_8 0x00000000 /* charge pump (mult. 1-8) */
310
#define HIFN_PLL_IS_9_12 0x00010000 /* charge pump (mult. 9-12) */
312
#define HIFN_PLL_FCK_MAX 266 /* Maximum PLL frequency */
314
/* Public key reset register (HIFN_1_PUB_RESET) */
315
#define HIFN_PUBRST_RESET 0x00000001 /* reset public/rng unit */
317
/* Public base address register (HIFN_1_PUB_BASE) */
318
#define HIFN_PUBBASE_ADDR 0x00003fff /* base address */
320
/* Public operand length register (HIFN_1_PUB_OPLEN) */
321
#define HIFN_PUBOPLEN_MOD_M 0x0000007f /* modulus length mask */
322
#define HIFN_PUBOPLEN_MOD_S 0 /* modulus length shift */
323
#define HIFN_PUBOPLEN_EXP_M 0x0003ff80 /* exponent length mask */
324
#define HIFN_PUBOPLEN_EXP_S 7 /* exponent length shift */
325
#define HIFN_PUBOPLEN_RED_M 0x003c0000 /* reducend length mask */
326
#define HIFN_PUBOPLEN_RED_S 18 /* reducend length shift */
328
/* Public operation register (HIFN_1_PUB_OP) */
329
#define HIFN_PUBOP_AOFFSET_M 0x0000007f /* A offset mask */
330
#define HIFN_PUBOP_AOFFSET_S 0 /* A offset shift */
331
#define HIFN_PUBOP_BOFFSET_M 0x00000f80 /* B offset mask */
332
#define HIFN_PUBOP_BOFFSET_S 7 /* B offset shift */
333
#define HIFN_PUBOP_MOFFSET_M 0x0003f000 /* M offset mask */
334
#define HIFN_PUBOP_MOFFSET_S 12 /* M offset shift */
335
#define HIFN_PUBOP_OP_MASK 0x003c0000 /* Opcode: */
336
#define HIFN_PUBOP_OP_NOP 0x00000000 /* NOP */
337
#define HIFN_PUBOP_OP_ADD 0x00040000 /* ADD */
338
#define HIFN_PUBOP_OP_ADDC 0x00080000 /* ADD w/carry */
339
#define HIFN_PUBOP_OP_SUB 0x000c0000 /* SUB */
340
#define HIFN_PUBOP_OP_SUBC 0x00100000 /* SUB w/carry */
341
#define HIFN_PUBOP_OP_MODADD 0x00140000 /* Modular ADD */
342
#define HIFN_PUBOP_OP_MODSUB 0x00180000 /* Modular SUB */
343
#define HIFN_PUBOP_OP_INCA 0x001c0000 /* INC A */
344
#define HIFN_PUBOP_OP_DECA 0x00200000 /* DEC A */
345
#define HIFN_PUBOP_OP_MULT 0x00240000 /* MULT */
346
#define HIFN_PUBOP_OP_MODMULT 0x00280000 /* Modular MULT */
347
#define HIFN_PUBOP_OP_MODRED 0x002c0000 /* Modular RED */
348
#define HIFN_PUBOP_OP_MODEXP 0x00300000 /* Modular EXP */
350
/* Public status register (HIFN_1_PUB_STATUS) */
351
#define HIFN_PUBSTS_DONE 0x00000001 /* operation done */
352
#define HIFN_PUBSTS_CARRY 0x00000002 /* carry */
354
/* Public interrupt enable register (HIFN_1_PUB_IEN) */
355
#define HIFN_PUBIEN_DONE 0x00000001 /* operation done interrupt */
357
/* Random number generator config register (HIFN_1_RNG_CONFIG) */
358
#define HIFN_RNGCFG_ENA 0x00000001 /* enable rng */
360
#define HIFN_NAMESIZE 32
361
#define HIFN_MAX_RESULT_ORDER 5
363
#define HIFN_D_CMD_RSIZE 24*1
364
#define HIFN_D_SRC_RSIZE 80*1
365
#define HIFN_D_DST_RSIZE 80*1
366
#define HIFN_D_RES_RSIZE 24*1
368
#define HIFN_D_DST_DALIGN 4
370
#define HIFN_QUEUE_LENGTH (HIFN_D_CMD_RSIZE - 1)
372
#define AES_MIN_KEY_SIZE 16
373
#define AES_MAX_KEY_SIZE 32
375
#define HIFN_DES_KEY_LENGTH 8
376
#define HIFN_3DES_KEY_LENGTH 24
377
#define HIFN_MAX_CRYPT_KEY_LENGTH AES_MAX_KEY_SIZE
378
#define HIFN_IV_LENGTH 8
379
#define HIFN_AES_IV_LENGTH 16
380
#define HIFN_MAX_IV_LENGTH HIFN_AES_IV_LENGTH
382
#define HIFN_MAC_KEY_LENGTH 64
383
#define HIFN_MD5_LENGTH 16
384
#define HIFN_SHA1_LENGTH 20
385
#define HIFN_MAC_TRUNC_LENGTH 12
387
#define HIFN_MAX_COMMAND (8 + 8 + 8 + 64 + 260)
388
#define HIFN_MAX_RESULT (8 + 4 + 4 + 20 + 4)
389
#define HIFN_USED_RESULT 12
398
struct hifn_desc cmdr[HIFN_D_CMD_RSIZE+1];
399
struct hifn_desc srcr[HIFN_D_SRC_RSIZE+1];
400
struct hifn_desc dstr[HIFN_D_DST_RSIZE+1];
401
struct hifn_desc resr[HIFN_D_RES_RSIZE+1];
403
u8 command_bufs[HIFN_D_CMD_RSIZE][HIFN_MAX_COMMAND];
404
u8 result_bufs[HIFN_D_CMD_RSIZE][HIFN_MAX_RESULT];
407
* Our current positions for insertion and removal from the descriptor
410
volatile int cmdi, srci, dsti, resi;
411
volatile int cmdu, srcu, dstu, resu;
412
int cmdk, srck, dstk, resk;
415
#define HIFN_FLAG_CMD_BUSY (1<<0)
416
#define HIFN_FLAG_SRC_BUSY (1<<1)
417
#define HIFN_FLAG_DST_BUSY (1<<2)
418
#define HIFN_FLAG_RES_BUSY (1<<3)
419
#define HIFN_FLAG_OLD_KEY (1<<4)
421
#define HIFN_DEFAULT_ACTIVE_NUM 5
425
char name[HIFN_NAMESIZE];
429
struct pci_dev *pdev;
430
void __iomem *bar[3];
437
void *sa[HIFN_D_RES_RSIZE];
443
struct delayed_work work;
445
unsigned long success;
446
unsigned long prev_success;
450
struct tasklet_struct tasklet;
452
struct crypto_queue queue;
453
struct list_head alg_list;
455
unsigned int pk_clk_freq;
457
#ifdef CONFIG_CRYPTO_DEV_HIFN_795X_RNG
458
unsigned int rng_wait_time;
464
#define HIFN_D_LENGTH 0x0000ffff
465
#define HIFN_D_NOINVALID 0x01000000
466
#define HIFN_D_MASKDONEIRQ 0x02000000
467
#define HIFN_D_DESTOVER 0x04000000
468
#define HIFN_D_OVER 0x08000000
469
#define HIFN_D_LAST 0x20000000
470
#define HIFN_D_JUMP 0x40000000
471
#define HIFN_D_VALID 0x80000000
473
struct hifn_base_command
475
volatile __le16 masks;
476
volatile __le16 session_num;
477
volatile __le16 total_source_count;
478
volatile __le16 total_dest_count;
481
#define HIFN_BASE_CMD_COMP 0x0100 /* enable compression engine */
482
#define HIFN_BASE_CMD_PAD 0x0200 /* enable padding engine */
483
#define HIFN_BASE_CMD_MAC 0x0400 /* enable MAC engine */
484
#define HIFN_BASE_CMD_CRYPT 0x0800 /* enable crypt engine */
485
#define HIFN_BASE_CMD_DECODE 0x2000
486
#define HIFN_BASE_CMD_SRCLEN_M 0xc000
487
#define HIFN_BASE_CMD_SRCLEN_S 14
488
#define HIFN_BASE_CMD_DSTLEN_M 0x3000
489
#define HIFN_BASE_CMD_DSTLEN_S 12
490
#define HIFN_BASE_CMD_LENMASK_HI 0x30000
491
#define HIFN_BASE_CMD_LENMASK_LO 0x0ffff
494
* Structure to help build up the command data structure.
496
struct hifn_crypt_command
498
volatile __le16 masks;
499
volatile __le16 header_skip;
500
volatile __le16 source_count;
501
volatile __le16 reserved;
504
#define HIFN_CRYPT_CMD_ALG_MASK 0x0003 /* algorithm: */
505
#define HIFN_CRYPT_CMD_ALG_DES 0x0000 /* DES */
506
#define HIFN_CRYPT_CMD_ALG_3DES 0x0001 /* 3DES */
507
#define HIFN_CRYPT_CMD_ALG_RC4 0x0002 /* RC4 */
508
#define HIFN_CRYPT_CMD_ALG_AES 0x0003 /* AES */
509
#define HIFN_CRYPT_CMD_MODE_MASK 0x0018 /* Encrypt mode: */
510
#define HIFN_CRYPT_CMD_MODE_ECB 0x0000 /* ECB */
511
#define HIFN_CRYPT_CMD_MODE_CBC 0x0008 /* CBC */
512
#define HIFN_CRYPT_CMD_MODE_CFB 0x0010 /* CFB */
513
#define HIFN_CRYPT_CMD_MODE_OFB 0x0018 /* OFB */
514
#define HIFN_CRYPT_CMD_CLR_CTX 0x0040 /* clear context */
515
#define HIFN_CRYPT_CMD_KSZ_MASK 0x0600 /* AES key size: */
516
#define HIFN_CRYPT_CMD_KSZ_128 0x0000 /* 128 bit */
517
#define HIFN_CRYPT_CMD_KSZ_192 0x0200 /* 192 bit */
518
#define HIFN_CRYPT_CMD_KSZ_256 0x0400 /* 256 bit */
519
#define HIFN_CRYPT_CMD_NEW_KEY 0x0800 /* expect new key */
520
#define HIFN_CRYPT_CMD_NEW_IV 0x1000 /* expect new iv */
521
#define HIFN_CRYPT_CMD_SRCLEN_M 0xc000
522
#define HIFN_CRYPT_CMD_SRCLEN_S 14
525
* Structure to help build up the command data structure.
527
struct hifn_mac_command
529
volatile __le16 masks;
530
volatile __le16 header_skip;
531
volatile __le16 source_count;
532
volatile __le16 reserved;
535
#define HIFN_MAC_CMD_ALG_MASK 0x0001
536
#define HIFN_MAC_CMD_ALG_SHA1 0x0000
537
#define HIFN_MAC_CMD_ALG_MD5 0x0001
538
#define HIFN_MAC_CMD_MODE_MASK 0x000c
539
#define HIFN_MAC_CMD_MODE_HMAC 0x0000
540
#define HIFN_MAC_CMD_MODE_SSL_MAC 0x0004
541
#define HIFN_MAC_CMD_MODE_HASH 0x0008
542
#define HIFN_MAC_CMD_MODE_FULL 0x0004
543
#define HIFN_MAC_CMD_TRUNC 0x0010
544
#define HIFN_MAC_CMD_RESULT 0x0020
545
#define HIFN_MAC_CMD_APPEND 0x0040
546
#define HIFN_MAC_CMD_SRCLEN_M 0xc000
547
#define HIFN_MAC_CMD_SRCLEN_S 14
550
* MAC POS IPsec initiates authentication after encryption on encodes
551
* and before decryption on decodes.
553
#define HIFN_MAC_CMD_POS_IPSEC 0x0200
554
#define HIFN_MAC_CMD_NEW_KEY 0x0800
556
struct hifn_comp_command
558
volatile __le16 masks;
559
volatile __le16 header_skip;
560
volatile __le16 source_count;
561
volatile __le16 reserved;
564
#define HIFN_COMP_CMD_SRCLEN_M 0xc000
565
#define HIFN_COMP_CMD_SRCLEN_S 14
566
#define HIFN_COMP_CMD_ONE 0x0100 /* must be one */
567
#define HIFN_COMP_CMD_CLEARHIST 0x0010 /* clear history */
568
#define HIFN_COMP_CMD_UPDATEHIST 0x0008 /* update history */
569
#define HIFN_COMP_CMD_LZS_STRIP0 0x0004 /* LZS: strip zero */
570
#define HIFN_COMP_CMD_MPPC_RESTART 0x0004 /* MPPC: restart */
571
#define HIFN_COMP_CMD_ALG_MASK 0x0001 /* compression mode: */
572
#define HIFN_COMP_CMD_ALG_MPPC 0x0001 /* MPPC */
573
#define HIFN_COMP_CMD_ALG_LZS 0x0000 /* LZS */
575
struct hifn_base_result
577
volatile __le16 flags;
578
volatile __le16 session;
579
volatile __le16 src_cnt; /* 15:0 of source count */
580
volatile __le16 dst_cnt; /* 15:0 of dest count */
583
#define HIFN_BASE_RES_DSTOVERRUN 0x0200 /* destination overrun */
584
#define HIFN_BASE_RES_SRCLEN_M 0xc000 /* 17:16 of source count */
585
#define HIFN_BASE_RES_SRCLEN_S 14
586
#define HIFN_BASE_RES_DSTLEN_M 0x3000 /* 17:16 of dest count */
587
#define HIFN_BASE_RES_DSTLEN_S 12
589
struct hifn_comp_result
591
volatile __le16 flags;
595
#define HIFN_COMP_RES_LCB_M 0xff00 /* longitudinal check byte */
596
#define HIFN_COMP_RES_LCB_S 8
597
#define HIFN_COMP_RES_RESTART 0x0004 /* MPPC: restart */
598
#define HIFN_COMP_RES_ENDMARKER 0x0002 /* LZS: end marker seen */
599
#define HIFN_COMP_RES_SRC_NOTZERO 0x0001 /* source expired */
601
struct hifn_mac_result
603
volatile __le16 flags;
604
volatile __le16 reserved;
605
/* followed by 0, 6, 8, or 10 u16's of the MAC, then crypt */
608
#define HIFN_MAC_RES_MISCOMPARE 0x0002 /* compare failed */
609
#define HIFN_MAC_RES_SRC_NOTZERO 0x0001 /* source expired */
611
struct hifn_crypt_result
613
volatile __le16 flags;
614
volatile __le16 reserved;
617
#define HIFN_CRYPT_RES_SRC_NOTZERO 0x0001 /* source expired */
619
#ifndef HIFN_POLL_FREQUENCY
620
#define HIFN_POLL_FREQUENCY 0x1
623
#ifndef HIFN_POLL_SCALAR
624
#define HIFN_POLL_SCALAR 0x0
627
#define HIFN_MAX_SEGLEN 0xffff /* maximum dma segment len */
628
#define HIFN_MAX_DMALEN 0x3ffff /* maximum dma length */
630
struct hifn_crypto_alg
632
struct list_head entry;
633
struct crypto_alg alg;
634
struct hifn_device *dev;
637
#define ASYNC_SCATTERLIST_CACHE 16
639
#define ASYNC_FLAGS_MISALIGNED (1<<0)
641
struct hifn_cipher_walk
643
struct scatterlist cache[ASYNC_SCATTERLIST_CACHE];
650
u8 key[HIFN_MAX_CRYPT_KEY_LENGTH];
651
struct hifn_device *dev;
652
unsigned int keysize;
655
struct hifn_request_context
659
u8 op, type, mode, unused;
660
struct hifn_cipher_walk walk;
663
#define crypto_alg_to_hifn(a) container_of(a, struct hifn_crypto_alg, alg)
665
static inline u32 hifn_read_0(struct hifn_device *dev, u32 reg)
669
ret = readl(dev->bar[0] + reg);
674
static inline u32 hifn_read_1(struct hifn_device *dev, u32 reg)
678
ret = readl(dev->bar[1] + reg);
683
static inline void hifn_write_0(struct hifn_device *dev, u32 reg, u32 val)
685
writel((__force u32)cpu_to_le32(val), dev->bar[0] + reg);
688
static inline void hifn_write_1(struct hifn_device *dev, u32 reg, u32 val)
690
writel((__force u32)cpu_to_le32(val), dev->bar[1] + reg);
693
static void hifn_wait_puc(struct hifn_device *dev)
698
for (i=10000; i > 0; --i) {
699
ret = hifn_read_0(dev, HIFN_0_PUCTRL);
700
if (!(ret & HIFN_PUCTRL_RESET))
707
dprintk("%s: Failed to reset PUC unit.\n", dev->name);
710
static void hifn_reset_puc(struct hifn_device *dev)
712
hifn_write_0(dev, HIFN_0_PUCTRL, HIFN_PUCTRL_DMAENA);
716
static void hifn_stop_device(struct hifn_device *dev)
718
hifn_write_1(dev, HIFN_1_DMA_CSR,
719
HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS |
720
HIFN_DMACSR_S_CTRL_DIS | HIFN_DMACSR_C_CTRL_DIS);
721
hifn_write_0(dev, HIFN_0_PUIER, 0);
722
hifn_write_1(dev, HIFN_1_DMA_IER, 0);
725
static void hifn_reset_dma(struct hifn_device *dev, int full)
727
hifn_stop_device(dev);
730
* Setting poll frequency and others to 0.
732
hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
733
HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
740
hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MODE);
743
hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MODE |
744
HIFN_DMACNFG_MSTRESET);
748
hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
749
HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
754
static u32 hifn_next_signature(u_int32_t a, u_int cnt)
759
for (i = 0; i < cnt; i++) {
769
a = (v & 1) ^ (a << 1);
775
static struct pci2id {
782
PCI_DEVICE_ID_HIFN_7955,
783
{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
784
0x00, 0x00, 0x00, 0x00, 0x00 }
788
PCI_DEVICE_ID_HIFN_7956,
789
{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
790
0x00, 0x00, 0x00, 0x00, 0x00 }
794
#ifdef CONFIG_CRYPTO_DEV_HIFN_795X_RNG
795
static int hifn_rng_data_present(struct hwrng *rng, int wait)
797
struct hifn_device *dev = (struct hifn_device *)rng->priv;
800
nsec = ktime_to_ns(ktime_sub(ktime_get(), dev->rngtime));
801
nsec -= dev->rng_wait_time;
810
static int hifn_rng_data_read(struct hwrng *rng, u32 *data)
812
struct hifn_device *dev = (struct hifn_device *)rng->priv;
814
*data = hifn_read_1(dev, HIFN_1_RNG_DATA);
815
dev->rngtime = ktime_get();
819
static int hifn_register_rng(struct hifn_device *dev)
822
* We must wait at least 256 Pk_clk cycles between two reads of the rng.
824
dev->rng_wait_time = DIV_ROUND_UP(NSEC_PER_SEC, dev->pk_clk_freq) *
827
dev->rng.name = dev->name;
828
dev->rng.data_present = hifn_rng_data_present,
829
dev->rng.data_read = hifn_rng_data_read,
830
dev->rng.priv = (unsigned long)dev;
832
return hwrng_register(&dev->rng);
835
static void hifn_unregister_rng(struct hifn_device *dev)
837
hwrng_unregister(&dev->rng);
840
#define hifn_register_rng(dev) 0
841
#define hifn_unregister_rng(dev)
844
static int hifn_init_pubrng(struct hifn_device *dev)
848
hifn_write_1(dev, HIFN_1_PUB_RESET, hifn_read_1(dev, HIFN_1_PUB_RESET) |
851
for (i=100; i > 0; --i) {
854
if ((hifn_read_1(dev, HIFN_1_PUB_RESET) & HIFN_PUBRST_RESET) == 0)
859
dprintk("Chip %s: Failed to initialise public key engine.\n",
862
hifn_write_1(dev, HIFN_1_PUB_IEN, HIFN_PUBIEN_DONE);
863
dev->dmareg |= HIFN_DMAIER_PUBDONE;
864
hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg);
866
dprintk("Chip %s: Public key engine has been successfully "
867
"initialised.\n", dev->name);
874
hifn_write_1(dev, HIFN_1_RNG_CONFIG,
875
hifn_read_1(dev, HIFN_1_RNG_CONFIG) | HIFN_RNGCFG_ENA);
876
dprintk("Chip %s: RNG engine has been successfully initialised.\n",
879
#ifdef CONFIG_CRYPTO_DEV_HIFN_795X_RNG
880
/* First value must be discarded */
881
hifn_read_1(dev, HIFN_1_RNG_DATA);
882
dev->rngtime = ktime_get();
887
static int hifn_enable_crypto(struct hifn_device *dev)
893
for (i = 0; i < ARRAY_SIZE(pci2id); i++) {
894
if (pci2id[i].pci_vendor == dev->pdev->vendor &&
895
pci2id[i].pci_prod == dev->pdev->device) {
896
offtbl = pci2id[i].card_id;
901
if (offtbl == NULL) {
902
dprintk("Chip %s: Unknown card!\n", dev->name);
906
dmacfg = hifn_read_1(dev, HIFN_1_DMA_CNFG);
908
hifn_write_1(dev, HIFN_1_DMA_CNFG,
909
HIFN_DMACNFG_UNLOCK | HIFN_DMACNFG_MSTRESET |
910
HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
912
addr = hifn_read_1(dev, HIFN_1_UNLOCK_SECRET1);
914
hifn_write_1(dev, HIFN_1_UNLOCK_SECRET2, 0);
917
for (i=0; i<12; ++i) {
918
addr = hifn_next_signature(addr, offtbl[i] + 0x101);
919
hifn_write_1(dev, HIFN_1_UNLOCK_SECRET2, addr);
923
hifn_write_1(dev, HIFN_1_DMA_CNFG, dmacfg);
925
dprintk("Chip %s: %s.\n", dev->name, pci_name(dev->pdev));
930
static void hifn_init_dma(struct hifn_device *dev)
932
struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
933
u32 dptr = dev->desc_dma;
936
for (i=0; i<HIFN_D_CMD_RSIZE; ++i)
937
dma->cmdr[i].p = __cpu_to_le32(dptr +
938
offsetof(struct hifn_dma, command_bufs[i][0]));
939
for (i=0; i<HIFN_D_RES_RSIZE; ++i)
940
dma->resr[i].p = __cpu_to_le32(dptr +
941
offsetof(struct hifn_dma, result_bufs[i][0]));
944
* Setup LAST descriptors.
946
dma->cmdr[HIFN_D_CMD_RSIZE].p = __cpu_to_le32(dptr +
947
offsetof(struct hifn_dma, cmdr[0]));
948
dma->srcr[HIFN_D_SRC_RSIZE].p = __cpu_to_le32(dptr +
949
offsetof(struct hifn_dma, srcr[0]));
950
dma->dstr[HIFN_D_DST_RSIZE].p = __cpu_to_le32(dptr +
951
offsetof(struct hifn_dma, dstr[0]));
952
dma->resr[HIFN_D_RES_RSIZE].p = __cpu_to_le32(dptr +
953
offsetof(struct hifn_dma, resr[0]));
955
dma->cmdu = dma->srcu = dma->dstu = dma->resu = 0;
956
dma->cmdi = dma->srci = dma->dsti = dma->resi = 0;
957
dma->cmdk = dma->srck = dma->dstk = dma->resk = 0;
961
* Initialize the PLL. We need to know the frequency of the reference clock
962
* to calculate the optimal multiplier. For PCI we assume 66MHz, since that
963
* allows us to operate without the risk of overclocking the chip. If it
964
* actually uses 33MHz, the chip will operate at half the speed, this can be
965
* overriden by specifying the frequency as module parameter (pci33).
967
* Unfortunately the PCI clock is not very suitable since the HIFN needs a
968
* stable clock and the PCI clock frequency may vary, so the default is the
969
* external clock. There is no way to find out its frequency, we default to
970
* 66MHz since according to Mike Ham of HiFn, almost every board in existence
971
* has an external crystal populated at 66MHz.
973
static void hifn_init_pll(struct hifn_device *dev)
975
unsigned int freq, m;
978
pllcfg = HIFN_1_PLL | HIFN_PLL_RESERVED_1;
980
if (strncmp(hifn_pll_ref, "ext", 3) == 0)
981
pllcfg |= HIFN_PLL_REF_CLK_PLL;
983
pllcfg |= HIFN_PLL_REF_CLK_HBI;
985
if (hifn_pll_ref[3] != '\0')
986
freq = simple_strtoul(hifn_pll_ref + 3, NULL, 10);
989
printk(KERN_INFO "hifn795x: assuming %uMHz clock speed, "
990
"override with hifn_pll_ref=%.3s<frequency>\n",
994
m = HIFN_PLL_FCK_MAX / freq;
996
pllcfg |= (m / 2 - 1) << HIFN_PLL_ND_SHIFT;
998
pllcfg |= HIFN_PLL_IS_1_8;
1000
pllcfg |= HIFN_PLL_IS_9_12;
1002
/* Select clock source and enable clock bypass */
1003
hifn_write_1(dev, HIFN_1_PLL, pllcfg |
1004
HIFN_PLL_PK_CLK_HBI | HIFN_PLL_PE_CLK_HBI | HIFN_PLL_BP);
1006
/* Let the chip lock to the input clock */
1009
/* Disable clock bypass */
1010
hifn_write_1(dev, HIFN_1_PLL, pllcfg |
1011
HIFN_PLL_PK_CLK_HBI | HIFN_PLL_PE_CLK_HBI);
1013
/* Switch the engines to the PLL */
1014
hifn_write_1(dev, HIFN_1_PLL, pllcfg |
1015
HIFN_PLL_PK_CLK_PLL | HIFN_PLL_PE_CLK_PLL);
1018
* The Fpk_clk runs at half the total speed. Its frequency is needed to
1019
* calculate the minimum time between two reads of the rng. Since 33MHz
1020
* is actually 33.333... we overestimate the frequency here, resulting
1021
* in slightly larger intervals.
1023
dev->pk_clk_freq = 1000000 * (freq + 1) * m / 2;
1026
static void hifn_init_registers(struct hifn_device *dev)
1028
u32 dptr = dev->desc_dma;
1030
/* Initialization magic... */
1031
hifn_write_0(dev, HIFN_0_PUCTRL, HIFN_PUCTRL_DMAENA);
1032
hifn_write_0(dev, HIFN_0_FIFOCNFG, HIFN_FIFOCNFG_THRESHOLD);
1033
hifn_write_0(dev, HIFN_0_PUIER, HIFN_PUIER_DSTOVER);
1035
/* write all 4 ring address registers */
1036
hifn_write_1(dev, HIFN_1_DMA_CRAR, dptr +
1037
offsetof(struct hifn_dma, cmdr[0]));
1038
hifn_write_1(dev, HIFN_1_DMA_SRAR, dptr +
1039
offsetof(struct hifn_dma, srcr[0]));
1040
hifn_write_1(dev, HIFN_1_DMA_DRAR, dptr +
1041
offsetof(struct hifn_dma, dstr[0]));
1042
hifn_write_1(dev, HIFN_1_DMA_RRAR, dptr +
1043
offsetof(struct hifn_dma, resr[0]));
1047
hifn_write_1(dev, HIFN_1_DMA_CSR,
1048
HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS |
1049
HIFN_DMACSR_S_CTRL_DIS | HIFN_DMACSR_C_CTRL_DIS |
1050
HIFN_DMACSR_D_ABORT | HIFN_DMACSR_D_DONE | HIFN_DMACSR_D_LAST |
1051
HIFN_DMACSR_D_WAIT | HIFN_DMACSR_D_OVER |
1052
HIFN_DMACSR_R_ABORT | HIFN_DMACSR_R_DONE | HIFN_DMACSR_R_LAST |
1053
HIFN_DMACSR_R_WAIT | HIFN_DMACSR_R_OVER |
1054
HIFN_DMACSR_S_ABORT | HIFN_DMACSR_S_DONE | HIFN_DMACSR_S_LAST |
1055
HIFN_DMACSR_S_WAIT |
1056
HIFN_DMACSR_C_ABORT | HIFN_DMACSR_C_DONE | HIFN_DMACSR_C_LAST |
1057
HIFN_DMACSR_C_WAIT |
1058
HIFN_DMACSR_ENGINE |
1059
HIFN_DMACSR_PUBDONE);
1061
hifn_write_1(dev, HIFN_1_DMA_CSR,
1062
HIFN_DMACSR_C_CTRL_ENA | HIFN_DMACSR_S_CTRL_ENA |
1063
HIFN_DMACSR_D_CTRL_ENA | HIFN_DMACSR_R_CTRL_ENA |
1064
HIFN_DMACSR_D_ABORT | HIFN_DMACSR_D_DONE | HIFN_DMACSR_D_LAST |
1065
HIFN_DMACSR_D_WAIT | HIFN_DMACSR_D_OVER |
1066
HIFN_DMACSR_R_ABORT | HIFN_DMACSR_R_DONE | HIFN_DMACSR_R_LAST |
1067
HIFN_DMACSR_R_WAIT | HIFN_DMACSR_R_OVER |
1068
HIFN_DMACSR_S_ABORT | HIFN_DMACSR_S_DONE | HIFN_DMACSR_S_LAST |
1069
HIFN_DMACSR_S_WAIT |
1070
HIFN_DMACSR_C_ABORT | HIFN_DMACSR_C_DONE | HIFN_DMACSR_C_LAST |
1071
HIFN_DMACSR_C_WAIT |
1072
HIFN_DMACSR_ENGINE |
1073
HIFN_DMACSR_PUBDONE);
1075
hifn_read_1(dev, HIFN_1_DMA_CSR);
1077
dev->dmareg |= HIFN_DMAIER_R_DONE | HIFN_DMAIER_C_ABORT |
1078
HIFN_DMAIER_D_OVER | HIFN_DMAIER_R_OVER |
1079
HIFN_DMAIER_S_ABORT | HIFN_DMAIER_D_ABORT | HIFN_DMAIER_R_ABORT |
1081
dev->dmareg &= ~HIFN_DMAIER_C_WAIT;
1083
hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg);
1084
hifn_read_1(dev, HIFN_1_DMA_IER);
1086
hifn_write_0(dev, HIFN_0_PUCNFG, HIFN_PUCNFG_ENCCNFG |
1087
HIFN_PUCNFG_DRFR_128 | HIFN_PUCNFG_TCALLPHASES |
1088
HIFN_PUCNFG_TCDRVTOTEM | HIFN_PUCNFG_BUS32 |
1091
hifn_write_0(dev, HIFN_0_PUCNFG, 0x10342);
1095
hifn_write_0(dev, HIFN_0_PUISR, HIFN_PUISR_DSTOVER);
1096
hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
1097
HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE | HIFN_DMACNFG_LAST |
1098
((HIFN_POLL_FREQUENCY << 16 ) & HIFN_DMACNFG_POLLFREQ) |
1099
((HIFN_POLL_SCALAR << 8) & HIFN_DMACNFG_POLLINVAL));
1102
static int hifn_setup_base_command(struct hifn_device *dev, u8 *buf,
1103
unsigned dlen, unsigned slen, u16 mask, u8 snum)
1105
struct hifn_base_command *base_cmd;
1108
base_cmd = (struct hifn_base_command *)buf_pos;
1109
base_cmd->masks = __cpu_to_le16(mask);
1110
base_cmd->total_source_count =
1111
__cpu_to_le16(slen & HIFN_BASE_CMD_LENMASK_LO);
1112
base_cmd->total_dest_count =
1113
__cpu_to_le16(dlen & HIFN_BASE_CMD_LENMASK_LO);
1117
base_cmd->session_num = __cpu_to_le16(snum |
1118
((slen << HIFN_BASE_CMD_SRCLEN_S) & HIFN_BASE_CMD_SRCLEN_M) |
1119
((dlen << HIFN_BASE_CMD_DSTLEN_S) & HIFN_BASE_CMD_DSTLEN_M));
1121
return sizeof(struct hifn_base_command);
1124
static int hifn_setup_crypto_command(struct hifn_device *dev,
1125
u8 *buf, unsigned dlen, unsigned slen,
1126
u8 *key, int keylen, u8 *iv, int ivsize, u16 mode)
1128
struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
1129
struct hifn_crypt_command *cry_cmd;
1133
cry_cmd = (struct hifn_crypt_command *)buf_pos;
1135
cry_cmd->source_count = __cpu_to_le16(dlen & 0xffff);
1137
cry_cmd->masks = __cpu_to_le16(mode |
1138
((dlen << HIFN_CRYPT_CMD_SRCLEN_S) &
1139
HIFN_CRYPT_CMD_SRCLEN_M));
1140
cry_cmd->header_skip = 0;
1141
cry_cmd->reserved = 0;
1143
buf_pos += sizeof(struct hifn_crypt_command);
1146
if (dma->cmdu > 1) {
1147
dev->dmareg |= HIFN_DMAIER_C_WAIT;
1148
hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg);
1152
memcpy(buf_pos, key, keylen);
1156
memcpy(buf_pos, iv, ivsize);
1160
cmd_len = buf_pos - buf;
1165
static int hifn_setup_cmd_desc(struct hifn_device *dev,
1166
struct hifn_context *ctx, struct hifn_request_context *rctx,
1167
void *priv, unsigned int nbytes)
1169
struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
1170
int cmd_len, sa_idx;
1175
buf_pos = buf = dma->command_bufs[dma->cmdi];
1179
case ACRYPTO_OP_DECRYPT:
1180
mask = HIFN_BASE_CMD_CRYPT | HIFN_BASE_CMD_DECODE;
1182
case ACRYPTO_OP_ENCRYPT:
1183
mask = HIFN_BASE_CMD_CRYPT;
1185
case ACRYPTO_OP_HMAC:
1186
mask = HIFN_BASE_CMD_MAC;
1192
buf_pos += hifn_setup_base_command(dev, buf_pos, nbytes,
1193
nbytes, mask, dev->snum);
1195
if (rctx->op == ACRYPTO_OP_ENCRYPT || rctx->op == ACRYPTO_OP_DECRYPT) {
1199
md |= HIFN_CRYPT_CMD_NEW_KEY;
1200
if (rctx->iv && rctx->mode != ACRYPTO_MODE_ECB)
1201
md |= HIFN_CRYPT_CMD_NEW_IV;
1203
switch (rctx->mode) {
1204
case ACRYPTO_MODE_ECB:
1205
md |= HIFN_CRYPT_CMD_MODE_ECB;
1207
case ACRYPTO_MODE_CBC:
1208
md |= HIFN_CRYPT_CMD_MODE_CBC;
1210
case ACRYPTO_MODE_CFB:
1211
md |= HIFN_CRYPT_CMD_MODE_CFB;
1213
case ACRYPTO_MODE_OFB:
1214
md |= HIFN_CRYPT_CMD_MODE_OFB;
1220
switch (rctx->type) {
1221
case ACRYPTO_TYPE_AES_128:
1222
if (ctx->keysize != 16)
1224
md |= HIFN_CRYPT_CMD_KSZ_128 |
1225
HIFN_CRYPT_CMD_ALG_AES;
1227
case ACRYPTO_TYPE_AES_192:
1228
if (ctx->keysize != 24)
1230
md |= HIFN_CRYPT_CMD_KSZ_192 |
1231
HIFN_CRYPT_CMD_ALG_AES;
1233
case ACRYPTO_TYPE_AES_256:
1234
if (ctx->keysize != 32)
1236
md |= HIFN_CRYPT_CMD_KSZ_256 |
1237
HIFN_CRYPT_CMD_ALG_AES;
1239
case ACRYPTO_TYPE_3DES:
1240
if (ctx->keysize != 24)
1242
md |= HIFN_CRYPT_CMD_ALG_3DES;
1244
case ACRYPTO_TYPE_DES:
1245
if (ctx->keysize != 8)
1247
md |= HIFN_CRYPT_CMD_ALG_DES;
1253
buf_pos += hifn_setup_crypto_command(dev, buf_pos,
1254
nbytes, nbytes, ctx->key, ctx->keysize,
1255
rctx->iv, rctx->ivsize, md);
1258
dev->sa[sa_idx] = priv;
1261
cmd_len = buf_pos - buf;
1262
dma->cmdr[dma->cmdi].l = __cpu_to_le32(cmd_len | HIFN_D_VALID |
1263
HIFN_D_LAST | HIFN_D_MASKDONEIRQ);
1265
if (++dma->cmdi == HIFN_D_CMD_RSIZE) {
1266
dma->cmdr[dma->cmdi].l = __cpu_to_le32(
1267
HIFN_D_VALID | HIFN_D_LAST |
1268
HIFN_D_MASKDONEIRQ | HIFN_D_JUMP);
1271
dma->cmdr[dma->cmdi-1].l |= __cpu_to_le32(HIFN_D_VALID);
1273
if (!(dev->flags & HIFN_FLAG_CMD_BUSY)) {
1274
hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_C_CTRL_ENA);
1275
dev->flags |= HIFN_FLAG_CMD_BUSY;
1283
static int hifn_setup_src_desc(struct hifn_device *dev, struct page *page,
1284
unsigned int offset, unsigned int size, int last)
1286
struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
1290
addr = pci_map_page(dev->pdev, page, offset, size, PCI_DMA_TODEVICE);
1294
dma->srcr[idx].p = __cpu_to_le32(addr);
1295
dma->srcr[idx].l = __cpu_to_le32(size | HIFN_D_VALID |
1296
HIFN_D_MASKDONEIRQ | (last ? HIFN_D_LAST : 0));
1298
if (++idx == HIFN_D_SRC_RSIZE) {
1299
dma->srcr[idx].l = __cpu_to_le32(HIFN_D_VALID |
1300
HIFN_D_JUMP | HIFN_D_MASKDONEIRQ |
1301
(last ? HIFN_D_LAST : 0));
1308
if (!(dev->flags & HIFN_FLAG_SRC_BUSY)) {
1309
hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_S_CTRL_ENA);
1310
dev->flags |= HIFN_FLAG_SRC_BUSY;
1316
static void hifn_setup_res_desc(struct hifn_device *dev)
1318
struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
1320
dma->resr[dma->resi].l = __cpu_to_le32(HIFN_USED_RESULT |
1321
HIFN_D_VALID | HIFN_D_LAST);
1323
* dma->resr[dma->resi].l = __cpu_to_le32(HIFN_MAX_RESULT | HIFN_D_VALID |
1327
if (++dma->resi == HIFN_D_RES_RSIZE) {
1328
dma->resr[HIFN_D_RES_RSIZE].l = __cpu_to_le32(HIFN_D_VALID |
1329
HIFN_D_JUMP | HIFN_D_MASKDONEIRQ | HIFN_D_LAST);
1335
if (!(dev->flags & HIFN_FLAG_RES_BUSY)) {
1336
hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_R_CTRL_ENA);
1337
dev->flags |= HIFN_FLAG_RES_BUSY;
1341
static void hifn_setup_dst_desc(struct hifn_device *dev, struct page *page,
1342
unsigned offset, unsigned size, int last)
1344
struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
1348
addr = pci_map_page(dev->pdev, page, offset, size, PCI_DMA_FROMDEVICE);
1351
dma->dstr[idx].p = __cpu_to_le32(addr);
1352
dma->dstr[idx].l = __cpu_to_le32(size | HIFN_D_VALID |
1353
HIFN_D_MASKDONEIRQ | (last ? HIFN_D_LAST : 0));
1355
if (++idx == HIFN_D_DST_RSIZE) {
1356
dma->dstr[idx].l = __cpu_to_le32(HIFN_D_VALID |
1357
HIFN_D_JUMP | HIFN_D_MASKDONEIRQ |
1358
(last ? HIFN_D_LAST : 0));
1364
if (!(dev->flags & HIFN_FLAG_DST_BUSY)) {
1365
hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_D_CTRL_ENA);
1366
dev->flags |= HIFN_FLAG_DST_BUSY;
1370
static int hifn_setup_dma(struct hifn_device *dev,
1371
struct hifn_context *ctx, struct hifn_request_context *rctx,
1372
struct scatterlist *src, struct scatterlist *dst,
1373
unsigned int nbytes, void *priv)
1375
struct scatterlist *t;
1376
struct page *spage, *dpage;
1377
unsigned int soff, doff;
1378
unsigned int n, len;
1382
spage = sg_page(src);
1384
len = min(src->length, n);
1386
hifn_setup_src_desc(dev, spage, soff, len, n - len == 0);
1392
t = &rctx->walk.cache[0];
1395
if (t->length && rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) {
1396
BUG_ON(!sg_page(t));
1401
BUG_ON(!sg_page(dst));
1402
dpage = sg_page(dst);
1408
hifn_setup_dst_desc(dev, dpage, doff, len, n - len == 0);
1415
hifn_setup_cmd_desc(dev, ctx, rctx, priv, nbytes);
1416
hifn_setup_res_desc(dev);
1420
static int hifn_cipher_walk_init(struct hifn_cipher_walk *w,
1421
int num, gfp_t gfp_flags)
1425
num = min(ASYNC_SCATTERLIST_CACHE, num);
1426
sg_init_table(w->cache, num);
1429
for (i=0; i<num; ++i) {
1430
struct page *page = alloc_page(gfp_flags);
1431
struct scatterlist *s;
1438
sg_set_page(s, page, PAGE_SIZE, 0);
1445
static void hifn_cipher_walk_exit(struct hifn_cipher_walk *w)
1449
for (i=0; i<w->num; ++i) {
1450
struct scatterlist *s = &w->cache[i];
1452
__free_page(sg_page(s));
1460
static int ablkcipher_add(unsigned int *drestp, struct scatterlist *dst,
1461
unsigned int size, unsigned int *nbytesp)
1463
unsigned int copy, drest = *drestp, nbytes = *nbytesp;
1466
if (drest < size || size > nbytes)
1470
copy = min3(drest, size, dst->length);
1476
dprintk("%s: copy: %u, size: %u, drest: %u, nbytes: %u.\n",
1477
__func__, copy, size, drest, nbytes);
1489
static int hifn_cipher_walk(struct ablkcipher_request *req,
1490
struct hifn_cipher_walk *w)
1492
struct scatterlist *dst, *t;
1493
unsigned int nbytes = req->nbytes, offset, copy, diff;
1499
if (idx >= w->num && (w->flags & ASYNC_FLAGS_MISALIGNED))
1502
dst = &req->dst[idx];
1504
dprintk("\n%s: dlen: %u, doff: %u, offset: %u, nbytes: %u.\n",
1505
__func__, dst->length, dst->offset, offset, nbytes);
1507
if (!IS_ALIGNED(dst->offset, HIFN_D_DST_DALIGN) ||
1508
!IS_ALIGNED(dst->length, HIFN_D_DST_DALIGN) ||
1510
unsigned slen = min(dst->length - offset, nbytes);
1511
unsigned dlen = PAGE_SIZE;
1515
err = ablkcipher_add(&dlen, dst, slen, &nbytes);
1521
copy = slen & ~(HIFN_D_DST_DALIGN - 1);
1522
diff = slen & (HIFN_D_DST_DALIGN - 1);
1524
if (dlen < nbytes) {
1526
* Destination page does not have enough space
1527
* to put there additional blocksized chunk,
1528
* so we mark that page as containing only
1529
* blocksize aligned chunks:
1530
* t->length = (slen & ~(HIFN_D_DST_DALIGN - 1));
1531
* and increase number of bytes to be processed
1538
* Temporary of course...
1539
* Kick author if you will catch this one.
1541
printk(KERN_ERR "%s: dlen: %u, nbytes: %u,"
1542
"slen: %u, offset: %u.\n",
1543
__func__, dlen, nbytes, slen, offset);
1544
printk(KERN_ERR "%s: please contact author to fix this "
1545
"issue, generally you should not catch "
1546
"this path under any condition but who "
1547
"knows how did you use crypto code.\n"
1548
"Thank you.\n", __func__);
1551
copy += diff + nbytes;
1553
dst = &req->dst[idx];
1555
err = ablkcipher_add(&dlen, dst, nbytes, &nbytes);
1565
nbytes -= min(dst->length, nbytes);
1575
static int hifn_setup_session(struct ablkcipher_request *req)
1577
struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm);
1578
struct hifn_request_context *rctx = ablkcipher_request_ctx(req);
1579
struct hifn_device *dev = ctx->dev;
1580
unsigned long dlen, flags;
1581
unsigned int nbytes = req->nbytes, idx = 0;
1582
int err = -EINVAL, sg_num;
1583
struct scatterlist *dst;
1585
if (rctx->iv && !rctx->ivsize && rctx->mode != ACRYPTO_MODE_ECB)
1588
rctx->walk.flags = 0;
1591
dst = &req->dst[idx];
1592
dlen = min(dst->length, nbytes);
1594
if (!IS_ALIGNED(dst->offset, HIFN_D_DST_DALIGN) ||
1595
!IS_ALIGNED(dlen, HIFN_D_DST_DALIGN))
1596
rctx->walk.flags |= ASYNC_FLAGS_MISALIGNED;
1602
if (rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) {
1603
err = hifn_cipher_walk_init(&rctx->walk, idx, GFP_ATOMIC);
1608
sg_num = hifn_cipher_walk(req, &rctx->walk);
1614
spin_lock_irqsave(&dev->lock, flags);
1615
if (dev->started + sg_num > HIFN_QUEUE_LENGTH) {
1620
err = hifn_setup_dma(dev, ctx, rctx, req->src, req->dst, req->nbytes, req);
1626
dev->active = HIFN_DEFAULT_ACTIVE_NUM;
1627
spin_unlock_irqrestore(&dev->lock, flags);
1632
spin_unlock_irqrestore(&dev->lock, flags);
1635
printk("%s: iv: %p [%d], key: %p [%d], mode: %u, op: %u, "
1636
"type: %u, err: %d.\n",
1637
dev->name, rctx->iv, rctx->ivsize,
1638
ctx->key, ctx->keysize,
1639
rctx->mode, rctx->op, rctx->type, err);
1645
static int hifn_test(struct hifn_device *dev, int encdec, u8 snum)
1649
struct hifn_context ctx;
1650
struct hifn_request_context rctx;
1651
u8 fips_aes_ecb_from_zero[16] = {
1652
0x66, 0xE9, 0x4B, 0xD4,
1653
0xEF, 0x8A, 0x2C, 0x3B,
1654
0x88, 0x4C, 0xFA, 0x59,
1655
0xCA, 0x34, 0x2B, 0x2E};
1656
struct scatterlist sg;
1658
memset(src, 0, sizeof(src));
1659
memset(ctx.key, 0, sizeof(ctx.key));
1665
rctx.op = (encdec)?ACRYPTO_OP_ENCRYPT:ACRYPTO_OP_DECRYPT;
1666
rctx.mode = ACRYPTO_MODE_ECB;
1667
rctx.type = ACRYPTO_TYPE_AES_128;
1668
rctx.walk.cache[0].length = 0;
1670
sg_init_one(&sg, &src, sizeof(src));
1672
err = hifn_setup_dma(dev, &ctx, &rctx, &sg, &sg, sizeof(src), NULL);
1679
dprintk("%s: decoded: ", dev->name);
1680
for (n=0; n<sizeof(src); ++n)
1681
dprintk("%02x ", src[n]);
1683
dprintk("%s: FIPS : ", dev->name);
1684
for (n=0; n<sizeof(fips_aes_ecb_from_zero); ++n)
1685
dprintk("%02x ", fips_aes_ecb_from_zero[n]);
1688
if (!memcmp(src, fips_aes_ecb_from_zero, sizeof(fips_aes_ecb_from_zero))) {
1689
printk(KERN_INFO "%s: AES 128 ECB test has been successfully "
1690
"passed.\n", dev->name);
1695
printk(KERN_INFO "%s: AES 128 ECB test has been failed.\n", dev->name);
1699
static int hifn_start_device(struct hifn_device *dev)
1703
dev->started = dev->active = 0;
1704
hifn_reset_dma(dev, 1);
1706
err = hifn_enable_crypto(dev);
1710
hifn_reset_puc(dev);
1714
hifn_init_registers(dev);
1716
hifn_init_pubrng(dev);
1721
static int ablkcipher_get(void *saddr, unsigned int *srestp, unsigned int offset,
1722
struct scatterlist *dst, unsigned int size, unsigned int *nbytesp)
1724
unsigned int srest = *srestp, nbytes = *nbytesp, copy;
1728
if (srest < size || size > nbytes)
1732
copy = min3(srest, dst->length, size);
1734
daddr = kmap_atomic(sg_page(dst), KM_IRQ0);
1735
memcpy(daddr + dst->offset + offset, saddr, copy);
1736
kunmap_atomic(daddr, KM_IRQ0);
1744
dprintk("%s: copy: %u, size: %u, srest: %u, nbytes: %u.\n",
1745
__func__, copy, size, srest, nbytes);
1757
static inline void hifn_complete_sa(struct hifn_device *dev, int i)
1759
unsigned long flags;
1761
spin_lock_irqsave(&dev->lock, flags);
1764
if (dev->started < 0)
1765
printk("%s: started: %d.\n", __func__, dev->started);
1766
spin_unlock_irqrestore(&dev->lock, flags);
1767
BUG_ON(dev->started < 0);
1770
static void hifn_process_ready(struct ablkcipher_request *req, int error)
1772
struct hifn_request_context *rctx = ablkcipher_request_ctx(req);
1774
if (rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) {
1775
unsigned int nbytes = req->nbytes;
1777
struct scatterlist *dst, *t;
1781
t = &rctx->walk.cache[idx];
1782
dst = &req->dst[idx];
1784
dprintk("\n%s: sg_page(t): %p, t->length: %u, "
1785
"sg_page(dst): %p, dst->length: %u, "
1787
__func__, sg_page(t), t->length,
1788
sg_page(dst), dst->length, nbytes);
1791
nbytes -= min(dst->length, nbytes);
1796
saddr = kmap_atomic(sg_page(t), KM_SOFTIRQ0);
1798
err = ablkcipher_get(saddr, &t->length, t->offset,
1799
dst, nbytes, &nbytes);
1801
kunmap_atomic(saddr, KM_SOFTIRQ0);
1806
kunmap_atomic(saddr, KM_SOFTIRQ0);
1809
hifn_cipher_walk_exit(&rctx->walk);
1812
req->base.complete(&req->base, error);
1815
static void hifn_clear_rings(struct hifn_device *dev, int error)
1817
struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
1820
dprintk("%s: ring cleanup 1: i: %d.%d.%d.%d, u: %d.%d.%d.%d, "
1821
"k: %d.%d.%d.%d.\n",
1823
dma->cmdi, dma->srci, dma->dsti, dma->resi,
1824
dma->cmdu, dma->srcu, dma->dstu, dma->resu,
1825
dma->cmdk, dma->srck, dma->dstk, dma->resk);
1827
i = dma->resk; u = dma->resu;
1829
if (dma->resr[i].l & __cpu_to_le32(HIFN_D_VALID))
1835
hifn_process_ready(dev->sa[i], error);
1836
hifn_complete_sa(dev, i);
1839
if (++i == HIFN_D_RES_RSIZE)
1843
dma->resk = i; dma->resu = u;
1845
i = dma->srck; u = dma->srcu;
1847
if (dma->srcr[i].l & __cpu_to_le32(HIFN_D_VALID))
1849
if (++i == HIFN_D_SRC_RSIZE)
1853
dma->srck = i; dma->srcu = u;
1855
i = dma->cmdk; u = dma->cmdu;
1857
if (dma->cmdr[i].l & __cpu_to_le32(HIFN_D_VALID))
1859
if (++i == HIFN_D_CMD_RSIZE)
1863
dma->cmdk = i; dma->cmdu = u;
1865
i = dma->dstk; u = dma->dstu;
1867
if (dma->dstr[i].l & __cpu_to_le32(HIFN_D_VALID))
1869
if (++i == HIFN_D_DST_RSIZE)
1873
dma->dstk = i; dma->dstu = u;
1875
dprintk("%s: ring cleanup 2: i: %d.%d.%d.%d, u: %d.%d.%d.%d, "
1876
"k: %d.%d.%d.%d.\n",
1878
dma->cmdi, dma->srci, dma->dsti, dma->resi,
1879
dma->cmdu, dma->srcu, dma->dstu, dma->resu,
1880
dma->cmdk, dma->srck, dma->dstk, dma->resk);
1883
static void hifn_work(struct work_struct *work)
1885
struct delayed_work *dw = to_delayed_work(work);
1886
struct hifn_device *dev = container_of(dw, struct hifn_device, work);
1887
unsigned long flags;
1891
spin_lock_irqsave(&dev->lock, flags);
1892
if (dev->active == 0) {
1893
struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
1895
if (dma->cmdu == 0 && (dev->flags & HIFN_FLAG_CMD_BUSY)) {
1896
dev->flags &= ~HIFN_FLAG_CMD_BUSY;
1897
r |= HIFN_DMACSR_C_CTRL_DIS;
1899
if (dma->srcu == 0 && (dev->flags & HIFN_FLAG_SRC_BUSY)) {
1900
dev->flags &= ~HIFN_FLAG_SRC_BUSY;
1901
r |= HIFN_DMACSR_S_CTRL_DIS;
1903
if (dma->dstu == 0 && (dev->flags & HIFN_FLAG_DST_BUSY)) {
1904
dev->flags &= ~HIFN_FLAG_DST_BUSY;
1905
r |= HIFN_DMACSR_D_CTRL_DIS;
1907
if (dma->resu == 0 && (dev->flags & HIFN_FLAG_RES_BUSY)) {
1908
dev->flags &= ~HIFN_FLAG_RES_BUSY;
1909
r |= HIFN_DMACSR_R_CTRL_DIS;
1912
hifn_write_1(dev, HIFN_1_DMA_CSR, r);
1916
if ((dev->prev_success == dev->success) && dev->started)
1918
dev->prev_success = dev->success;
1919
spin_unlock_irqrestore(&dev->lock, flags);
1922
if (++dev->reset >= 5) {
1924
struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
1926
printk("%s: r: %08x, active: %d, started: %d, "
1927
"success: %lu: qlen: %u/%u, reset: %d.\n",
1928
dev->name, r, dev->active, dev->started,
1929
dev->success, dev->queue.qlen, dev->queue.max_qlen,
1932
printk("%s: res: ", __func__);
1933
for (i=0; i<HIFN_D_RES_RSIZE; ++i) {
1934
printk("%x.%p ", dma->resr[i].l, dev->sa[i]);
1936
hifn_process_ready(dev->sa[i], -ENODEV);
1937
hifn_complete_sa(dev, i);
1942
hifn_reset_dma(dev, 1);
1943
hifn_stop_device(dev);
1944
hifn_start_device(dev);
1948
tasklet_schedule(&dev->tasklet);
1951
schedule_delayed_work(&dev->work, HZ);
1954
static irqreturn_t hifn_interrupt(int irq, void *data)
1956
struct hifn_device *dev = (struct hifn_device *)data;
1957
struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
1958
u32 dmacsr, restart;
1960
dmacsr = hifn_read_1(dev, HIFN_1_DMA_CSR);
1962
dprintk("%s: 1 dmacsr: %08x, dmareg: %08x, res: %08x [%d], "
1963
"i: %d.%d.%d.%d, u: %d.%d.%d.%d.\n",
1964
dev->name, dmacsr, dev->dmareg, dmacsr & dev->dmareg, dma->cmdi,
1965
dma->cmdi, dma->srci, dma->dsti, dma->resi,
1966
dma->cmdu, dma->srcu, dma->dstu, dma->resu);
1968
if ((dmacsr & dev->dmareg) == 0)
1971
hifn_write_1(dev, HIFN_1_DMA_CSR, dmacsr & dev->dmareg);
1973
if (dmacsr & HIFN_DMACSR_ENGINE)
1974
hifn_write_0(dev, HIFN_0_PUISR, hifn_read_0(dev, HIFN_0_PUISR));
1975
if (dmacsr & HIFN_DMACSR_PUBDONE)
1976
hifn_write_1(dev, HIFN_1_PUB_STATUS,
1977
hifn_read_1(dev, HIFN_1_PUB_STATUS) | HIFN_PUBSTS_DONE);
1979
restart = dmacsr & (HIFN_DMACSR_R_OVER | HIFN_DMACSR_D_OVER);
1981
u32 puisr = hifn_read_0(dev, HIFN_0_PUISR);
1983
printk(KERN_WARNING "%s: overflow: r: %d, d: %d, puisr: %08x, d: %u.\n",
1984
dev->name, !!(dmacsr & HIFN_DMACSR_R_OVER),
1985
!!(dmacsr & HIFN_DMACSR_D_OVER),
1986
puisr, !!(puisr & HIFN_PUISR_DSTOVER));
1987
if (!!(puisr & HIFN_PUISR_DSTOVER))
1988
hifn_write_0(dev, HIFN_0_PUISR, HIFN_PUISR_DSTOVER);
1989
hifn_write_1(dev, HIFN_1_DMA_CSR, dmacsr & (HIFN_DMACSR_R_OVER |
1990
HIFN_DMACSR_D_OVER));
1993
restart = dmacsr & (HIFN_DMACSR_C_ABORT | HIFN_DMACSR_S_ABORT |
1994
HIFN_DMACSR_D_ABORT | HIFN_DMACSR_R_ABORT);
1996
printk(KERN_WARNING "%s: abort: c: %d, s: %d, d: %d, r: %d.\n",
1997
dev->name, !!(dmacsr & HIFN_DMACSR_C_ABORT),
1998
!!(dmacsr & HIFN_DMACSR_S_ABORT),
1999
!!(dmacsr & HIFN_DMACSR_D_ABORT),
2000
!!(dmacsr & HIFN_DMACSR_R_ABORT));
2001
hifn_reset_dma(dev, 1);
2003
hifn_init_registers(dev);
2006
if ((dmacsr & HIFN_DMACSR_C_WAIT) && (dma->cmdu == 0)) {
2007
dprintk("%s: wait on command.\n", dev->name);
2008
dev->dmareg &= ~(HIFN_DMAIER_C_WAIT);
2009
hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg);
2012
tasklet_schedule(&dev->tasklet);
2017
static void hifn_flush(struct hifn_device *dev)
2019
unsigned long flags;
2020
struct crypto_async_request *async_req;
2021
struct ablkcipher_request *req;
2022
struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
2025
for (i=0; i<HIFN_D_RES_RSIZE; ++i) {
2026
struct hifn_desc *d = &dma->resr[i];
2029
hifn_process_ready(dev->sa[i],
2030
(d->l & __cpu_to_le32(HIFN_D_VALID))?-ENODEV:0);
2031
hifn_complete_sa(dev, i);
2035
spin_lock_irqsave(&dev->lock, flags);
2036
while ((async_req = crypto_dequeue_request(&dev->queue))) {
2037
req = container_of(async_req, struct ablkcipher_request, base);
2038
spin_unlock_irqrestore(&dev->lock, flags);
2040
hifn_process_ready(req, -ENODEV);
2042
spin_lock_irqsave(&dev->lock, flags);
2044
spin_unlock_irqrestore(&dev->lock, flags);
2047
static int hifn_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
2050
struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
2051
struct hifn_context *ctx = crypto_tfm_ctx(tfm);
2052
struct hifn_device *dev = ctx->dev;
2054
if (len > HIFN_MAX_CRYPT_KEY_LENGTH) {
2055
crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
2059
if (len == HIFN_DES_KEY_LENGTH) {
2060
u32 tmp[DES_EXPKEY_WORDS];
2061
int ret = des_ekey(tmp, key);
2063
if (unlikely(ret == 0) && (tfm->crt_flags & CRYPTO_TFM_REQ_WEAK_KEY)) {
2064
tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY;
2069
dev->flags &= ~HIFN_FLAG_OLD_KEY;
2071
memcpy(ctx->key, key, len);
2077
static int hifn_handle_req(struct ablkcipher_request *req)
2079
struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm);
2080
struct hifn_device *dev = ctx->dev;
2083
if (dev->started + DIV_ROUND_UP(req->nbytes, PAGE_SIZE) <= HIFN_QUEUE_LENGTH)
2084
err = hifn_setup_session(req);
2086
if (err == -EAGAIN) {
2087
unsigned long flags;
2089
spin_lock_irqsave(&dev->lock, flags);
2090
err = ablkcipher_enqueue_request(&dev->queue, req);
2091
spin_unlock_irqrestore(&dev->lock, flags);
2097
static int hifn_setup_crypto_req(struct ablkcipher_request *req, u8 op,
2100
struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm);
2101
struct hifn_request_context *rctx = ablkcipher_request_ctx(req);
2104
ivsize = crypto_ablkcipher_ivsize(crypto_ablkcipher_reqtfm(req));
2106
if (req->info && mode != ACRYPTO_MODE_ECB) {
2107
if (type == ACRYPTO_TYPE_AES_128)
2108
ivsize = HIFN_AES_IV_LENGTH;
2109
else if (type == ACRYPTO_TYPE_DES)
2110
ivsize = HIFN_DES_KEY_LENGTH;
2111
else if (type == ACRYPTO_TYPE_3DES)
2112
ivsize = HIFN_3DES_KEY_LENGTH;
2115
if (ctx->keysize != 16 && type == ACRYPTO_TYPE_AES_128) {
2116
if (ctx->keysize == 24)
2117
type = ACRYPTO_TYPE_AES_192;
2118
else if (ctx->keysize == 32)
2119
type = ACRYPTO_TYPE_AES_256;
2125
rctx->iv = req->info;
2126
rctx->ivsize = ivsize;
2129
* HEAVY TODO: needs to kick Herbert XU to write documentation.
2130
* HEAVY TODO: needs to kick Herbert XU to write documentation.
2131
* HEAVY TODO: needs to kick Herbert XU to write documentation.
2134
return hifn_handle_req(req);
2137
static int hifn_process_queue(struct hifn_device *dev)
2139
struct crypto_async_request *async_req, *backlog;
2140
struct ablkcipher_request *req;
2141
unsigned long flags;
2144
while (dev->started < HIFN_QUEUE_LENGTH) {
2145
spin_lock_irqsave(&dev->lock, flags);
2146
backlog = crypto_get_backlog(&dev->queue);
2147
async_req = crypto_dequeue_request(&dev->queue);
2148
spin_unlock_irqrestore(&dev->lock, flags);
2154
backlog->complete(backlog, -EINPROGRESS);
2156
req = container_of(async_req, struct ablkcipher_request, base);
2158
err = hifn_handle_req(req);
2166
static int hifn_setup_crypto(struct ablkcipher_request *req, u8 op,
2170
struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm);
2171
struct hifn_device *dev = ctx->dev;
2173
err = hifn_setup_crypto_req(req, op, type, mode);
2177
if (dev->started < HIFN_QUEUE_LENGTH && dev->queue.qlen)
2178
hifn_process_queue(dev);
2180
return -EINPROGRESS;
2184
* AES ecryption functions.
2186
static inline int hifn_encrypt_aes_ecb(struct ablkcipher_request *req)
2188
return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2189
ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_ECB);
2191
static inline int hifn_encrypt_aes_cbc(struct ablkcipher_request *req)
2193
return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2194
ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_CBC);
2196
static inline int hifn_encrypt_aes_cfb(struct ablkcipher_request *req)
2198
return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2199
ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_CFB);
2201
static inline int hifn_encrypt_aes_ofb(struct ablkcipher_request *req)
2203
return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2204
ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_OFB);
2208
* AES decryption functions.
2210
static inline int hifn_decrypt_aes_ecb(struct ablkcipher_request *req)
2212
return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2213
ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_ECB);
2215
static inline int hifn_decrypt_aes_cbc(struct ablkcipher_request *req)
2217
return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2218
ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_CBC);
2220
static inline int hifn_decrypt_aes_cfb(struct ablkcipher_request *req)
2222
return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2223
ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_CFB);
2225
static inline int hifn_decrypt_aes_ofb(struct ablkcipher_request *req)
2227
return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2228
ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_OFB);
2232
* DES ecryption functions.
2234
static inline int hifn_encrypt_des_ecb(struct ablkcipher_request *req)
2236
return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2237
ACRYPTO_TYPE_DES, ACRYPTO_MODE_ECB);
2239
static inline int hifn_encrypt_des_cbc(struct ablkcipher_request *req)
2241
return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2242
ACRYPTO_TYPE_DES, ACRYPTO_MODE_CBC);
2244
static inline int hifn_encrypt_des_cfb(struct ablkcipher_request *req)
2246
return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2247
ACRYPTO_TYPE_DES, ACRYPTO_MODE_CFB);
2249
static inline int hifn_encrypt_des_ofb(struct ablkcipher_request *req)
2251
return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2252
ACRYPTO_TYPE_DES, ACRYPTO_MODE_OFB);
2256
* DES decryption functions.
2258
static inline int hifn_decrypt_des_ecb(struct ablkcipher_request *req)
2260
return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2261
ACRYPTO_TYPE_DES, ACRYPTO_MODE_ECB);
2263
static inline int hifn_decrypt_des_cbc(struct ablkcipher_request *req)
2265
return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2266
ACRYPTO_TYPE_DES, ACRYPTO_MODE_CBC);
2268
static inline int hifn_decrypt_des_cfb(struct ablkcipher_request *req)
2270
return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2271
ACRYPTO_TYPE_DES, ACRYPTO_MODE_CFB);
2273
static inline int hifn_decrypt_des_ofb(struct ablkcipher_request *req)
2275
return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2276
ACRYPTO_TYPE_DES, ACRYPTO_MODE_OFB);
2280
* 3DES ecryption functions.
2282
static inline int hifn_encrypt_3des_ecb(struct ablkcipher_request *req)
2284
return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2285
ACRYPTO_TYPE_3DES, ACRYPTO_MODE_ECB);
2287
static inline int hifn_encrypt_3des_cbc(struct ablkcipher_request *req)
2289
return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2290
ACRYPTO_TYPE_3DES, ACRYPTO_MODE_CBC);
2292
static inline int hifn_encrypt_3des_cfb(struct ablkcipher_request *req)
2294
return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2295
ACRYPTO_TYPE_3DES, ACRYPTO_MODE_CFB);
2297
static inline int hifn_encrypt_3des_ofb(struct ablkcipher_request *req)
2299
return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2300
ACRYPTO_TYPE_3DES, ACRYPTO_MODE_OFB);
2304
* 3DES decryption functions.
2306
static inline int hifn_decrypt_3des_ecb(struct ablkcipher_request *req)
2308
return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2309
ACRYPTO_TYPE_3DES, ACRYPTO_MODE_ECB);
2311
static inline int hifn_decrypt_3des_cbc(struct ablkcipher_request *req)
2313
return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2314
ACRYPTO_TYPE_3DES, ACRYPTO_MODE_CBC);
2316
static inline int hifn_decrypt_3des_cfb(struct ablkcipher_request *req)
2318
return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2319
ACRYPTO_TYPE_3DES, ACRYPTO_MODE_CFB);
2321
static inline int hifn_decrypt_3des_ofb(struct ablkcipher_request *req)
2323
return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2324
ACRYPTO_TYPE_3DES, ACRYPTO_MODE_OFB);
2327
struct hifn_alg_template
2329
char name[CRYPTO_MAX_ALG_NAME];
2330
char drv_name[CRYPTO_MAX_ALG_NAME];
2332
struct ablkcipher_alg ablkcipher;
2335
static struct hifn_alg_template hifn_alg_templates[] = {
2337
* 3DES ECB, CBC, CFB and OFB modes.
2340
.name = "cfb(des3_ede)", .drv_name = "cfb-3des", .bsize = 8,
2342
.min_keysize = HIFN_3DES_KEY_LENGTH,
2343
.max_keysize = HIFN_3DES_KEY_LENGTH,
2344
.setkey = hifn_setkey,
2345
.encrypt = hifn_encrypt_3des_cfb,
2346
.decrypt = hifn_decrypt_3des_cfb,
2350
.name = "ofb(des3_ede)", .drv_name = "ofb-3des", .bsize = 8,
2352
.min_keysize = HIFN_3DES_KEY_LENGTH,
2353
.max_keysize = HIFN_3DES_KEY_LENGTH,
2354
.setkey = hifn_setkey,
2355
.encrypt = hifn_encrypt_3des_ofb,
2356
.decrypt = hifn_decrypt_3des_ofb,
2360
.name = "cbc(des3_ede)", .drv_name = "cbc-3des", .bsize = 8,
2362
.ivsize = HIFN_IV_LENGTH,
2363
.min_keysize = HIFN_3DES_KEY_LENGTH,
2364
.max_keysize = HIFN_3DES_KEY_LENGTH,
2365
.setkey = hifn_setkey,
2366
.encrypt = hifn_encrypt_3des_cbc,
2367
.decrypt = hifn_decrypt_3des_cbc,
2371
.name = "ecb(des3_ede)", .drv_name = "ecb-3des", .bsize = 8,
2373
.min_keysize = HIFN_3DES_KEY_LENGTH,
2374
.max_keysize = HIFN_3DES_KEY_LENGTH,
2375
.setkey = hifn_setkey,
2376
.encrypt = hifn_encrypt_3des_ecb,
2377
.decrypt = hifn_decrypt_3des_ecb,
2382
* DES ECB, CBC, CFB and OFB modes.
2385
.name = "cfb(des)", .drv_name = "cfb-des", .bsize = 8,
2387
.min_keysize = HIFN_DES_KEY_LENGTH,
2388
.max_keysize = HIFN_DES_KEY_LENGTH,
2389
.setkey = hifn_setkey,
2390
.encrypt = hifn_encrypt_des_cfb,
2391
.decrypt = hifn_decrypt_des_cfb,
2395
.name = "ofb(des)", .drv_name = "ofb-des", .bsize = 8,
2397
.min_keysize = HIFN_DES_KEY_LENGTH,
2398
.max_keysize = HIFN_DES_KEY_LENGTH,
2399
.setkey = hifn_setkey,
2400
.encrypt = hifn_encrypt_des_ofb,
2401
.decrypt = hifn_decrypt_des_ofb,
2405
.name = "cbc(des)", .drv_name = "cbc-des", .bsize = 8,
2407
.ivsize = HIFN_IV_LENGTH,
2408
.min_keysize = HIFN_DES_KEY_LENGTH,
2409
.max_keysize = HIFN_DES_KEY_LENGTH,
2410
.setkey = hifn_setkey,
2411
.encrypt = hifn_encrypt_des_cbc,
2412
.decrypt = hifn_decrypt_des_cbc,
2416
.name = "ecb(des)", .drv_name = "ecb-des", .bsize = 8,
2418
.min_keysize = HIFN_DES_KEY_LENGTH,
2419
.max_keysize = HIFN_DES_KEY_LENGTH,
2420
.setkey = hifn_setkey,
2421
.encrypt = hifn_encrypt_des_ecb,
2422
.decrypt = hifn_decrypt_des_ecb,
2427
* AES ECB, CBC, CFB and OFB modes.
2430
.name = "ecb(aes)", .drv_name = "ecb-aes", .bsize = 16,
2432
.min_keysize = AES_MIN_KEY_SIZE,
2433
.max_keysize = AES_MAX_KEY_SIZE,
2434
.setkey = hifn_setkey,
2435
.encrypt = hifn_encrypt_aes_ecb,
2436
.decrypt = hifn_decrypt_aes_ecb,
2440
.name = "cbc(aes)", .drv_name = "cbc-aes", .bsize = 16,
2442
.ivsize = HIFN_AES_IV_LENGTH,
2443
.min_keysize = AES_MIN_KEY_SIZE,
2444
.max_keysize = AES_MAX_KEY_SIZE,
2445
.setkey = hifn_setkey,
2446
.encrypt = hifn_encrypt_aes_cbc,
2447
.decrypt = hifn_decrypt_aes_cbc,
2451
.name = "cfb(aes)", .drv_name = "cfb-aes", .bsize = 16,
2453
.min_keysize = AES_MIN_KEY_SIZE,
2454
.max_keysize = AES_MAX_KEY_SIZE,
2455
.setkey = hifn_setkey,
2456
.encrypt = hifn_encrypt_aes_cfb,
2457
.decrypt = hifn_decrypt_aes_cfb,
2461
.name = "ofb(aes)", .drv_name = "ofb-aes", .bsize = 16,
2463
.min_keysize = AES_MIN_KEY_SIZE,
2464
.max_keysize = AES_MAX_KEY_SIZE,
2465
.setkey = hifn_setkey,
2466
.encrypt = hifn_encrypt_aes_ofb,
2467
.decrypt = hifn_decrypt_aes_ofb,
2472
static int hifn_cra_init(struct crypto_tfm *tfm)
2474
struct crypto_alg *alg = tfm->__crt_alg;
2475
struct hifn_crypto_alg *ha = crypto_alg_to_hifn(alg);
2476
struct hifn_context *ctx = crypto_tfm_ctx(tfm);
2479
tfm->crt_ablkcipher.reqsize = sizeof(struct hifn_request_context);
2483
static int hifn_alg_alloc(struct hifn_device *dev, struct hifn_alg_template *t)
2485
struct hifn_crypto_alg *alg;
2488
alg = kzalloc(sizeof(struct hifn_crypto_alg), GFP_KERNEL);
2492
snprintf(alg->alg.cra_name, CRYPTO_MAX_ALG_NAME, "%s", t->name);
2493
snprintf(alg->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s-%s",
2494
t->drv_name, dev->name);
2496
alg->alg.cra_priority = 300;
2497
alg->alg.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC;
2498
alg->alg.cra_blocksize = t->bsize;
2499
alg->alg.cra_ctxsize = sizeof(struct hifn_context);
2500
alg->alg.cra_alignmask = 0;
2501
alg->alg.cra_type = &crypto_ablkcipher_type;
2502
alg->alg.cra_module = THIS_MODULE;
2503
alg->alg.cra_u.ablkcipher = t->ablkcipher;
2504
alg->alg.cra_init = hifn_cra_init;
2508
list_add_tail(&alg->entry, &dev->alg_list);
2510
err = crypto_register_alg(&alg->alg);
2512
list_del(&alg->entry);
2519
static void hifn_unregister_alg(struct hifn_device *dev)
2521
struct hifn_crypto_alg *a, *n;
2523
list_for_each_entry_safe(a, n, &dev->alg_list, entry) {
2524
list_del(&a->entry);
2525
crypto_unregister_alg(&a->alg);
2530
static int hifn_register_alg(struct hifn_device *dev)
2534
for (i=0; i<ARRAY_SIZE(hifn_alg_templates); ++i) {
2535
err = hifn_alg_alloc(dev, &hifn_alg_templates[i]);
2543
hifn_unregister_alg(dev);
2547
static void hifn_tasklet_callback(unsigned long data)
2549
struct hifn_device *dev = (struct hifn_device *)data;
2552
* This is ok to call this without lock being held,
2553
* althogh it modifies some parameters used in parallel,
2554
* (like dev->success), but they are used in process
2555
* context or update is atomic (like setting dev->sa[i] to NULL).
2557
hifn_clear_rings(dev, 0);
2559
if (dev->started < HIFN_QUEUE_LENGTH && dev->queue.qlen)
2560
hifn_process_queue(dev);
2563
static int __devinit hifn_probe(struct pci_dev *pdev, const struct pci_device_id *id)
2566
struct hifn_device *dev;
2569
err = pci_enable_device(pdev);
2572
pci_set_master(pdev);
2574
err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
2576
goto err_out_disable_pci_device;
2578
snprintf(name, sizeof(name), "hifn%d",
2579
atomic_inc_return(&hifn_dev_number)-1);
2581
err = pci_request_regions(pdev, name);
2583
goto err_out_disable_pci_device;
2585
if (pci_resource_len(pdev, 0) < HIFN_BAR0_SIZE ||
2586
pci_resource_len(pdev, 1) < HIFN_BAR1_SIZE ||
2587
pci_resource_len(pdev, 2) < HIFN_BAR2_SIZE) {
2588
dprintk("%s: Broken hardware - I/O regions are too small.\n",
2591
goto err_out_free_regions;
2594
dev = kzalloc(sizeof(struct hifn_device) + sizeof(struct crypto_alg),
2598
goto err_out_free_regions;
2601
INIT_LIST_HEAD(&dev->alg_list);
2603
snprintf(dev->name, sizeof(dev->name), "%s", name);
2604
spin_lock_init(&dev->lock);
2606
for (i=0; i<3; ++i) {
2607
unsigned long addr, size;
2609
addr = pci_resource_start(pdev, i);
2610
size = pci_resource_len(pdev, i);
2612
dev->bar[i] = ioremap_nocache(addr, size);
2614
goto err_out_unmap_bars;
2617
dev->desc_virt = pci_alloc_consistent(pdev, sizeof(struct hifn_dma),
2619
if (!dev->desc_virt) {
2620
dprintk("Failed to allocate descriptor rings.\n");
2621
goto err_out_unmap_bars;
2623
memset(dev->desc_virt, 0, sizeof(struct hifn_dma));
2626
dev->irq = pdev->irq;
2628
for (i=0; i<HIFN_D_RES_RSIZE; ++i)
2631
pci_set_drvdata(pdev, dev);
2633
tasklet_init(&dev->tasklet, hifn_tasklet_callback, (unsigned long)dev);
2635
crypto_init_queue(&dev->queue, 1);
2637
err = request_irq(dev->irq, hifn_interrupt, IRQF_SHARED, dev->name, dev);
2639
dprintk("Failed to request IRQ%d: err: %d.\n", dev->irq, err);
2641
goto err_out_free_desc;
2644
err = hifn_start_device(dev);
2646
goto err_out_free_irq;
2648
err = hifn_test(dev, 1, 0);
2650
goto err_out_stop_device;
2652
err = hifn_register_rng(dev);
2654
goto err_out_stop_device;
2656
err = hifn_register_alg(dev);
2658
goto err_out_unregister_rng;
2660
INIT_DELAYED_WORK(&dev->work, hifn_work);
2661
schedule_delayed_work(&dev->work, HZ);
2663
dprintk("HIFN crypto accelerator card at %s has been "
2664
"successfully registered as %s.\n",
2665
pci_name(pdev), dev->name);
2669
err_out_unregister_rng:
2670
hifn_unregister_rng(dev);
2671
err_out_stop_device:
2672
hifn_reset_dma(dev, 1);
2673
hifn_stop_device(dev);
2675
free_irq(dev->irq, dev->name);
2676
tasklet_kill(&dev->tasklet);
2678
pci_free_consistent(pdev, sizeof(struct hifn_dma),
2679
dev->desc_virt, dev->desc_dma);
2684
iounmap(dev->bar[i]);
2686
err_out_free_regions:
2687
pci_release_regions(pdev);
2689
err_out_disable_pci_device:
2690
pci_disable_device(pdev);
2695
static void __devexit hifn_remove(struct pci_dev *pdev)
2698
struct hifn_device *dev;
2700
dev = pci_get_drvdata(pdev);
2703
cancel_delayed_work_sync(&dev->work);
2705
hifn_unregister_rng(dev);
2706
hifn_unregister_alg(dev);
2707
hifn_reset_dma(dev, 1);
2708
hifn_stop_device(dev);
2710
free_irq(dev->irq, dev->name);
2711
tasklet_kill(&dev->tasklet);
2715
pci_free_consistent(pdev, sizeof(struct hifn_dma),
2716
dev->desc_virt, dev->desc_dma);
2719
iounmap(dev->bar[i]);
2724
pci_release_regions(pdev);
2725
pci_disable_device(pdev);
2728
static struct pci_device_id hifn_pci_tbl[] = {
2729
{ PCI_DEVICE(PCI_VENDOR_ID_HIFN, PCI_DEVICE_ID_HIFN_7955) },
2730
{ PCI_DEVICE(PCI_VENDOR_ID_HIFN, PCI_DEVICE_ID_HIFN_7956) },
2733
MODULE_DEVICE_TABLE(pci, hifn_pci_tbl);
2735
static struct pci_driver hifn_pci_driver = {
2737
.id_table = hifn_pci_tbl,
2738
.probe = hifn_probe,
2739
.remove = __devexit_p(hifn_remove),
2742
static int __init hifn_init(void)
2747
/* HIFN supports only 32-bit addresses */
2748
BUILD_BUG_ON(sizeof(dma_addr_t) != 4);
2750
if (strncmp(hifn_pll_ref, "ext", 3) &&
2751
strncmp(hifn_pll_ref, "pci", 3)) {
2752
printk(KERN_ERR "hifn795x: invalid hifn_pll_ref clock, "
2753
"must be pci or ext");
2758
* For the 7955/7956 the reference clock frequency must be in the
2759
* range of 20MHz-100MHz. For the 7954 the upper bound is 66.67MHz,
2760
* but this chip is currently not supported.
2762
if (hifn_pll_ref[3] != '\0') {
2763
freq = simple_strtoul(hifn_pll_ref + 3, NULL, 10);
2764
if (freq < 20 || freq > 100) {
2765
printk(KERN_ERR "hifn795x: invalid hifn_pll_ref "
2766
"frequency, must be in the range "
2772
err = pci_register_driver(&hifn_pci_driver);
2774
dprintk("Failed to register PCI driver for %s device.\n",
2775
hifn_pci_driver.name);
2779
printk(KERN_INFO "Driver for HIFN 795x crypto accelerator chip "
2780
"has been successfully registered.\n");
2785
static void __exit hifn_fini(void)
2787
pci_unregister_driver(&hifn_pci_driver);
2789
printk(KERN_INFO "Driver for HIFN 795x crypto accelerator chip "
2790
"has been successfully unregistered.\n");
2793
module_init(hifn_init);
2794
module_exit(hifn_fini);
2796
MODULE_LICENSE("GPL");
2797
MODULE_AUTHOR("Evgeniy Polyakov <johnpol@2ka.mipt.ru>");
2798
MODULE_DESCRIPTION("Driver for HIFN 795x crypto accelerator chip.");