ubsec.c source code [src/src/sys/dev/pci/ubsec.c]

1	/ $NetBSD: ubsec.c,v 1.43 2016/07/07 06:55:41 msaitoh Exp $ /
2	/ $FreeBSD: src/sys/dev/ubsec/ubsec.c,v 1.6.2.6 2003/01/23 21:06:43 sam Exp $ /
3	/ $OpenBSD: ubsec.c,v 1.143 2009/03/27 13:31:30 reyk Exp$ /
4
5	/*
6	* Copyright (c) 2000 Jason L. Wright (jason@thought.net)
7	* Copyright (c) 2000 Theo de Raadt (deraadt@openbsd.org)
8	* Copyright (c) 2001 Patrik Lindergren (patrik@ipunplugged.com)
9	*
10	* Redistribution and use in source and binary forms, with or without
11	* modification, are permitted provided that the following conditions
12	* are met:
13	* 1. Redistributions of source code must retain the above copyright
14	* notice, this list of conditions and the following disclaimer.
15	* 2. Redistributions in binary form must reproduce the above copyright
16	* notice, this list of conditions and the following disclaimer in the
17	* documentation and/or other materials provided with the distribution.
18	*
19	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
20	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
21	* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
22	* DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
23	* INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
24	* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
25	* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
27	* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
28	* ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29	* POSSIBILITY OF SUCH DAMAGE.
30	*
31	* Effort sponsored in part by the Defense Advanced Research Projects
32	* Agency (DARPA) and Air Force Research Laboratory, Air Force
33	* Materiel Command, USAF, under agreement number F30602-01-2-0537.
34	*
35	*/
36
37	#include <sys/cdefs.h>
38	__KERNEL_RCSID(`0`, "$NetBSD: ubsec.c,v 1.43 2016/07/07 06:55:41 msaitoh Exp $");
39
40	#undef UBSEC_DEBUG
41
42	/*
43	* uBsec 5[56]01, 58xx hardware crypto accelerator
44	*/
45
46	#include <sys/param.h>
47	#include <sys/systm.h>
48	#include <sys/proc.h>
49	#include <sys/endian.h>
50	#ifdef __NetBSD__
51	#define UBSEC_NO_RNG /* hangs on attach */
52	#define letoh16 htole16
53	#define letoh32 htole32
54	#endif
55	#include <sys/errno.h>
56	#include <sys/malloc.h>
57	#include <sys/kernel.h>
58	#include <sys/mbuf.h>
59	#include <sys/device.h>
60	#include <sys/module.h>
61	#include <sys/queue.h>
62	#include <sys/sysctl.h>
63
64	#include <opencrypto/cryptodev.h>
65	#include <opencrypto/xform.h>
66	#ifdef __OpenBSD__
67	#include <dev/rndvar.h>
68	#include <sys/md5k.h>
69	#else
70	#include <sys/cprng.h>
71	#include <sys/md5.h>
72	#include <sys/rndpool.h>
73	#include <sys/rndsource.h>
74	#endif
75	#include <sys/sha1.h>
76
77	#include <dev/pci/pcireg.h>
78	#include <dev/pci/pcivar.h>
79	#include <dev/pci/pcidevs.h>
80
81	#include <dev/pci/ubsecreg.h>
82	#include <dev/pci/ubsecvar.h>
83
84	/*
85	* Prototypes and count for the pci_device structure
86	*/
87	static int ubsec_probe(device_t, cfdata_t, void *);
88	static void ubsec_attach(device_t, device_t, void *);
89	static int ubsec_detach(device_t, int);
90	static int ubsec_sysctl_init(void);
91	static void ubsec_reset_board(struct ubsec_softc *);
92	static void ubsec_init_board(struct ubsec_softc *);
93	static void ubsec_init_pciregs(struct pci_attach_args *pa);
94	static void ubsec_cleanchip(struct ubsec_softc *);
95	static void ubsec_totalreset(struct ubsec_softc *);
96	static int ubsec_free_q(struct ubsec_softc, struct* ubsec_q *);
97
98	#ifdef __OpenBSD__
99	struct cfattach ubsec_ca = {
100	sizeof(struct ubsec_softc), ubsec_probe, ubsec_attach,
101	};
102
103	struct cfdriver ubsec_cd = {
104	`0`, "ubsec", DV_DULL
105	};
106	#else
107	CFATTACH_DECL_NEW(ubsec, sizeof(struct ubsec_softc), ubsec_probe, ubsec_attach,
108	ubsec_detach, NULL);
109	extern struct cfdriver ubsec_cd;
110	#endif
111
112	/ patchable /
113	#ifdef UBSEC_DEBUG
114	extern int ubsec_debug;
115	int ubsec_debug=`1`;
116	#endif
117
118	static int ubsec_intr(void *);
119	static int ubsec_newsession(void, u_int32_t , struct cryptoini *);
120	static int ubsec_freesession(void*, u_int64_t);
121	static int ubsec_process(void, struct* cryptop , int* hint);
122	static void ubsec_callback(struct ubsec_softc , struct* ubsec_q *);
123	static void ubsec_feed(struct ubsec_softc *);
124	static void ubsec_mcopy(struct mbuf , struct* mbuf , int, int*);
125	static void ubsec_callback2(struct ubsec_softc , struct* ubsec_q2 *);
126	static void ubsec_feed2(struct ubsec_softc *);
127	static void ubsec_feed4(struct ubsec_softc *);
128	#ifndef UBSEC_NO_RNG
129	static void ubsec_rng(void *);
130	static void ubsec_rng_locked(void *);
131	static void ubsec_rng_get(size_t, void *);
132	#endif /* UBSEC_NO_RNG */
133	static int ubsec_dma_malloc(struct ubsec_softc *, bus_size_t,
134	struct ubsec_dma_alloc , int*);
135	static void ubsec_dma_free(struct ubsec_softc , struct* ubsec_dma_alloc *);
136	static int ubsec_dmamap_aligned(bus_dmamap_t);
137
138	static int ubsec_kprocess(void, struct* cryptkop , int*);
139	static int ubsec_kprocess_modexp_sw(struct ubsec_softc *,
140	struct cryptkop , int*);
141	static int ubsec_kprocess_modexp_hw(struct ubsec_softc *,
142	struct cryptkop , int*);
143	static int ubsec_kprocess_rsapriv(struct ubsec_softc *,
144	struct cryptkop , int*);
145	static void ubsec_kfree(struct ubsec_softc , struct* ubsec_q2 *);
146	static int ubsec_ksigbits(struct crparam *);
147	static void ubsec_kshift_r(u_int, u_int8_t , u_int, u_int8_t , u_int);
148	static void ubsec_kshift_l(u_int, u_int8_t , u_int, u_int8_t , u_int);
149
150	#ifdef UBSEC_DEBUG
151	static void ubsec_dump_pb(volatile struct ubsec_pktbuf *);
152	static void ubsec_dump_mcr(struct ubsec_mcr *);
153	static void ubsec_dump_ctx2(volatile struct ubsec_ctx_keyop *);
154	#endif
155
156	#define READ_REG(sc,r) \
157	bus_space_read_4((sc)->sc_st, (sc)->sc_sh, (r))
158
159	#define WRITE_REG(sc,reg,val) \
160	bus_space_write_4((sc)->sc_st, (sc)->sc_sh, reg, val)
161
162	#define SWAP32(x) (x) = htole32(ntohl((x)))
163	#ifndef HTOLE32
164	#define HTOLE32(x) (x) = htole32(x)
165	#endif
166
167	struct ubsec_stats ubsecstats;
168
169	static struct sysctllog *ubsec_sysctllog;
170
171	/*
172	* ubsec_maxbatch controls the number of crypto ops to voluntarily
173	* collect into one submission to the hardware. This batching happens
174	* when ops are dispatched from the crypto subsystem with a hint that
175	* more are to follow immediately. These ops must also not be marked
176	* with a ``no delay'' flag.
177	*/
178	static int ubsec_maxbatch = `1`;
179
180	/*
181	* ubsec_maxaggr controls the number of crypto ops to submit to the
182	* hardware as a unit. This aggregation reduces the number of interrupts
183	* to the host at the expense of increased latency (for all but the last
184	* operation). For network traffic setting this to one yields the highest
185	* performance but at the expense of more interrupt processing.
186	*/
187	static int ubsec_maxaggr = `1`;
188
189	static const struct ubsec_product {
190	pci_vendor_id_t ubsec_vendor;
191	pci_product_id_t ubsec_product;
192	int ubsec_flags;
193	int ubsec_statmask;
194	int ubsec_maxaggr;
195	const char *ubsec_name;
196	} ubsec_products[] = {
197	{ PCI_VENDOR_BLUESTEEL, PCI_PRODUCT_BLUESTEEL_5501,
198	`0`,
199	BS_STAT_MCR1_DONE \| BS_STAT_DMAERR,
200	UBS_MIN_AGGR,
201	"Bluesteel 5501"
202	},
203	{ PCI_VENDOR_BLUESTEEL, PCI_PRODUCT_BLUESTEEL_5601,
204	UBS_FLAGS_KEY \| UBS_FLAGS_RNG,
205	BS_STAT_MCR1_DONE \| BS_STAT_DMAERR,
206	UBS_MIN_AGGR,
207	"Bluesteel 5601"
208	},
209
210	{ PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5801,
211	`0`,
212	BS_STAT_MCR1_DONE \| BS_STAT_DMAERR,
213	UBS_MIN_AGGR,
214	"Broadcom BCM5801"
215	},
216
217	{ PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5802,
218	UBS_FLAGS_KEY \| UBS_FLAGS_RNG,
219	BS_STAT_MCR1_DONE \| BS_STAT_DMAERR,
220	UBS_MIN_AGGR,
221	"Broadcom BCM5802"
222	},
223
224	{ PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5805,
225	UBS_FLAGS_KEY \| UBS_FLAGS_RNG,
226	BS_STAT_MCR1_DONE \| BS_STAT_DMAERR,
227	UBS_MIN_AGGR,
228	"Broadcom BCM5805"
229	},
230
231	{ PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5820,
232	UBS_FLAGS_KEY \| UBS_FLAGS_RNG \| UBS_FLAGS_LONGCTX \|
233	UBS_FLAGS_HWNORM \| UBS_FLAGS_BIGKEY,
234	BS_STAT_MCR1_DONE \| BS_STAT_DMAERR,
235	UBS_MIN_AGGR,
236	"Broadcom BCM5820"
237	},
238
239	{ PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5821,
240	UBS_FLAGS_KEY \| UBS_FLAGS_RNG \| UBS_FLAGS_LONGCTX \|
241	UBS_FLAGS_HWNORM \| UBS_FLAGS_BIGKEY,
242	BS_STAT_MCR1_DONE \| BS_STAT_DMAERR \|
243	BS_STAT_MCR1_ALLEMPTY \| BS_STAT_MCR2_ALLEMPTY,
244	UBS_MIN_AGGR,
245	"Broadcom BCM5821"
246	},
247	{ PCI_VENDOR_SUN, PCI_PRODUCT_SUN_SCA1K,
248	UBS_FLAGS_KEY \| UBS_FLAGS_RNG \| UBS_FLAGS_LONGCTX \|
249	UBS_FLAGS_HWNORM \| UBS_FLAGS_BIGKEY,
250	BS_STAT_MCR1_DONE \| BS_STAT_DMAERR \|
251	BS_STAT_MCR1_ALLEMPTY \| BS_STAT_MCR2_ALLEMPTY,
252	UBS_MIN_AGGR,
253	"Sun Crypto Accelerator 1000"
254	},
255	{ PCI_VENDOR_SUN, PCI_PRODUCT_SUN_5821,
256	UBS_FLAGS_KEY \| UBS_FLAGS_RNG \| UBS_FLAGS_LONGCTX \|
257	UBS_FLAGS_HWNORM \| UBS_FLAGS_BIGKEY,
258	BS_STAT_MCR1_DONE \| BS_STAT_DMAERR \|
259	BS_STAT_MCR1_ALLEMPTY \| BS_STAT_MCR2_ALLEMPTY,
260	UBS_MIN_AGGR,
261	"Broadcom BCM5821 (Sun)"
262	},
263
264	{ PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5822,
265	UBS_FLAGS_KEY \| UBS_FLAGS_RNG \| UBS_FLAGS_LONGCTX \|
266	UBS_FLAGS_HWNORM \| UBS_FLAGS_BIGKEY,
267	BS_STAT_MCR1_DONE \| BS_STAT_DMAERR \|
268	BS_STAT_MCR1_ALLEMPTY \| BS_STAT_MCR2_ALLEMPTY,
269	UBS_MIN_AGGR,
270	"Broadcom BCM5822"
271	},
272
273	{ PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5823,
274	UBS_FLAGS_KEY \| UBS_FLAGS_RNG \| UBS_FLAGS_LONGCTX \|
275	UBS_FLAGS_HWNORM \| UBS_FLAGS_BIGKEY \| UBS_FLAGS_AES,
276	BS_STAT_MCR1_DONE \| BS_STAT_DMAERR \|
277	BS_STAT_MCR1_ALLEMPTY \| BS_STAT_MCR2_ALLEMPTY,
278	UBS_MIN_AGGR,
279	"Broadcom BCM5823"
280	},
281
282	{ PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5825,
283	UBS_FLAGS_KEY \| UBS_FLAGS_RNG \| UBS_FLAGS_LONGCTX \|
284	UBS_FLAGS_HWNORM \| UBS_FLAGS_BIGKEY \| UBS_FLAGS_AES,
285	BS_STAT_MCR1_DONE \| BS_STAT_DMAERR \|
286	BS_STAT_MCR1_ALLEMPTY \| BS_STAT_MCR2_ALLEMPTY,
287	UBS_MIN_AGGR,
288	"Broadcom BCM5825"
289	},
290
291	{ PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5860,
292	UBS_FLAGS_MULTIMCR \| UBS_FLAGS_HWNORM \|
293	UBS_FLAGS_LONGCTX \|
294	UBS_FLAGS_RNG \| UBS_FLAGS_RNG4 \|
295	UBS_FLAGS_KEY \| UBS_FLAGS_BIGKEY \| UBS_FLAGS_AES,
296	BS_STAT_MCR1_DONE \| BS_STAT_DMAERR \|
297	BS_STAT_MCR1_ALLEMPTY \| BS_STAT_MCR2_ALLEMPTY \|
298	BS_STAT_MCR3_ALLEMPTY \| BS_STAT_MCR4_ALLEMPTY,
299	UBS_MAX_AGGR,
300	"Broadcom BCM5860"
301	},
302
303	{ PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5861,
304	UBS_FLAGS_MULTIMCR \| UBS_FLAGS_HWNORM \|
305	UBS_FLAGS_LONGCTX \|
306	UBS_FLAGS_RNG \| UBS_FLAGS_RNG4 \|
307	UBS_FLAGS_KEY \| UBS_FLAGS_BIGKEY \| UBS_FLAGS_AES,
308	BS_STAT_MCR1_DONE \| BS_STAT_DMAERR \|
309	BS_STAT_MCR1_ALLEMPTY \| BS_STAT_MCR2_ALLEMPTY \|
310	BS_STAT_MCR3_ALLEMPTY \| BS_STAT_MCR4_ALLEMPTY,
311	UBS_MAX_AGGR,
312	"Broadcom BCM5861"
313	},
314
315	{ PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5862,
316	UBS_FLAGS_MULTIMCR \| UBS_FLAGS_HWNORM \|
317	UBS_FLAGS_LONGCTX \|
318	UBS_FLAGS_RNG \| UBS_FLAGS_RNG4 \|
319	UBS_FLAGS_KEY \| UBS_FLAGS_BIGKEY \| UBS_FLAGS_AES,
320	BS_STAT_MCR1_DONE \| BS_STAT_DMAERR \|
321	BS_STAT_MCR1_ALLEMPTY \| BS_STAT_MCR2_ALLEMPTY \|
322	BS_STAT_MCR3_ALLEMPTY \| BS_STAT_MCR4_ALLEMPTY,
323	UBS_MAX_AGGR,
324	"Broadcom BCM5862"
325	},
326
327	{ `0`, `0`,
328	`0`,
329	`0`,
330	`0`,
331	NULL
332	}
333	};
334
335	static const struct ubsec_product *
336	ubsec_lookup(const struct pci_attach_args *pa)
337	{
338	const struct ubsec_product *up;
339
340	for (up = ubsec_products; up->ubsec_name != NULL; up++) {
341	if (PCI_VENDOR(pa->pa_id) == up->ubsec_vendor &&
342	PCI_PRODUCT(pa->pa_id) == up->ubsec_product)
343	return (up);
344	}
345	return (NULL);
346	}
347
348	static int
349	ubsec_probe(device_t parent, cfdata_t match, void *aux)
350	{
351	struct pci_attach_args pa = (struct* pci_attach_args *)aux;
352
353	if (ubsec_lookup(pa) != NULL)
354	return (`1`);
355
356	return (`0`);
357	}
358
359	static void
360	ubsec_attach(device_t parent, device_t self, void *aux)
361	{
362	struct ubsec_softc *sc = device_private(self);
363	struct pci_attach_args *pa = aux;
364	const struct ubsec_product *up;
365	pci_chipset_tag_t pc = pa->pa_pc;
366	pci_intr_handle_t ih;
367	const char *intrstr = NULL;
368	pcireg_t memtype;
369	struct ubsec_dma *dmap;
370	u_int32_t cmd, i;
371	char intrbuf[PCI_INTRSTR_LEN];
372
373	sc->sc_dev = self;
374	sc->sc_pct = pc;
375
376	up = ubsec_lookup(pa);
377	if (up == NULL) {
378	printf("\n");
379	panic("ubsec_attach: impossible");
380	}
381
382	pci_aprint_devinfo_fancy(pa, "Crypto processor", up->ubsec_name, `1`);
383
384	SIMPLEQ_INIT(&sc->sc_queue);
385	SIMPLEQ_INIT(&sc->sc_qchip);
386	SIMPLEQ_INIT(&sc->sc_queue2);
387	SIMPLEQ_INIT(&sc->sc_qchip2);
388	SIMPLEQ_INIT(&sc->sc_queue4);
389	SIMPLEQ_INIT(&sc->sc_qchip4);
390	SIMPLEQ_INIT(&sc->sc_q2free);
391
392	sc->sc_flags = up->ubsec_flags;
393	sc->sc_statmask = up->ubsec_statmask;
394	sc->sc_maxaggr = up->ubsec_maxaggr;
395
396	cmd = pci_conf_read(pc, pa->pa_tag, PCI_COMMAND_STATUS_REG);
397	cmd \|= PCI_COMMAND_MASTER_ENABLE;
398	pci_conf_write(pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, cmd);
399
400	memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, BS_BAR);
401	if (pci_mapreg_map(pa, BS_BAR, memtype, `0`,
402	&sc->sc_st, &sc->sc_sh, NULL, &sc->sc_memsize)) {
403	aprint_error_dev(self, "can't find mem space");
404	return;
405	}
406
407	sc->sc_dmat = pa->pa_dmat;
408
409	if (pci_intr_map(pa, &ih)) {
410	aprint_error_dev(self, "couldn't map interrupt\n");
411	return;
412	}
413	intrstr = pci_intr_string(pc, ih, intrbuf, sizeof(intrbuf));
414	sc->sc_ih = pci_intr_establish(pc, ih, IPL_NET, ubsec_intr, sc);
415	if (sc->sc_ih == NULL) {
416	aprint_error_dev(self, "couldn't establish interrupt");
417	if (intrstr != NULL)
418	aprint_error(" at %s", intrstr);
419	aprint_error("\n");
420	return;
421	}
422	aprint_normal_dev(self, "interrupting at %s\n", intrstr);
423
424	sc->sc_cid = crypto_get_driverid(`0`);
425	if (sc->sc_cid < `0`) {
426	aprint_error_dev(self, "couldn't get crypto driver id\n");
427	pci_intr_disestablish(pc, sc->sc_ih);
428	return;
429	}
430
431	sc->sc_rng_need = RND_POOLBITS / NBBY;
432	mutex_init(&sc->sc_mtx, MUTEX_DEFAULT, IPL_VM);
433
434	SIMPLEQ_INIT(&sc->sc_freequeue);
435	dmap = sc->sc_dmaa;
436	for (i = `0`; i < UBS_MAX_NQUEUE; i++, dmap++) {
437	struct ubsec_q *q;
438
439	q = (struct ubsec_q )malloc(sizeof(struct* ubsec_q),
440	M_DEVBUF, M_ZERO\|M_NOWAIT);
441	if (q == NULL) {
442	aprint_error_dev(self,
443	"can't allocate queue buffers\n");
444	break;
445	}
446
447	if (ubsec_dma_malloc(sc, sizeof(struct ubsec_dmachunk),
448	&dmap->d_alloc, `0`)) {
449	aprint_error_dev(self, "can't allocate dma buffers\n");
450	free(q, M_DEVBUF);
451	break;
452	}
453	dmap->d_dma = (struct ubsec_dmachunk *)dmap->d_alloc.dma_vaddr;
454
455	q->q_dma = dmap;
456	sc->sc_queuea[i] = q;
457
458	SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
459	}
460
461	crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, `0`, `0`,
462	ubsec_newsession, ubsec_freesession, ubsec_process, sc);
463	crypto_register(sc->sc_cid, CRYPTO_DES_CBC, `0`, `0`,
464	ubsec_newsession, ubsec_freesession, ubsec_process, sc);
465	crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC_96, `0`, `0`,
466	ubsec_newsession, ubsec_freesession, ubsec_process, sc);
467	crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC_96, `0`, `0`,
468	ubsec_newsession, ubsec_freesession, ubsec_process, sc);
469	if (sc->sc_flags & UBS_FLAGS_AES) {
470	crypto_register(sc->sc_cid, CRYPTO_AES_CBC, `0`, `0`,
471	ubsec_newsession, ubsec_freesession, ubsec_process, sc);
472	}
473
474	/*
475	* Reset Broadcom chip
476	*/
477	ubsec_reset_board(sc);
478
479	/*
480	* Init Broadcom specific PCI settings
481	*/
482	ubsec_init_pciregs(pa);
483
484	/*
485	* Init Broadcom chip
486	*/
487	ubsec_init_board(sc);
488
489	#ifndef UBSEC_NO_RNG
490	if (sc->sc_flags & UBS_FLAGS_RNG) {
491	if (sc->sc_flags & UBS_FLAGS_RNG4)
492	sc->sc_statmask \|= BS_STAT_MCR4_DONE;
493	else
494	sc->sc_statmask \|= BS_STAT_MCR2_DONE;
495
496	if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
497	&sc->sc_rng.rng_q.q_mcr, `0`))
498	goto skip_rng;
499
500	if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rngbypass),
501	&sc->sc_rng.rng_q.q_ctx, `0`)) {
502	ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr);
503	goto skip_rng;
504	}
505
506	if (ubsec_dma_malloc(sc, sizeof(u_int32_t) *
507	UBSEC_RNG_BUFSIZ, &sc->sc_rng.rng_buf, `0`)) {
508	ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx);
509	ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr);
510	goto skip_rng;
511	}
512
513	rndsource_setcb(&sc->sc_rnd_source, ubsec_rng_get, sc);
514	rnd_attach_source(&sc->sc_rnd_source, device_xname(sc->sc_dev),
515	RND_TYPE_RNG,
516	RND_FLAG_COLLECT_VALUE\|RND_FLAG_HASCB);
517	if (hz >= `100`)
518	sc->sc_rnghz = hz / `100`;
519	else
520	sc->sc_rnghz = `1`;
521	#ifdef __OpenBSD__
522	timeout_set(&sc->sc_rngto, ubsec_rng, sc);
523	timeout_add(&sc->sc_rngto, sc->sc_rnghz);
524	#else
525	callout_init(&sc->sc_rngto, `0`);
526	callout_setfunc(&sc->sc_rngto, ubsec_rng, sc);
527	callout_schedule(&sc->sc_rngto, sc->sc_rnghz);
528	#endif
529	skip_rng:
530	if (sc->sc_rnghz)
531	aprint_normal_dev(self,
532	"random number generator enabled\n");
533	else
534	aprint_error_dev(self,
535	"WARNING: random number generator disabled\n");
536	}
537	#endif /* UBSEC_NO_RNG */
538
539	if (sc->sc_flags & UBS_FLAGS_KEY) {
540	sc->sc_statmask \|= BS_STAT_MCR2_DONE;
541
542	crypto_kregister(sc->sc_cid, CRK_MOD_EXP, `0`,
543	ubsec_kprocess, sc);
544	#if 0
545	crypto_kregister(sc->sc_cid, CRK_MOD_EXP_CRT, `0`,
546	ubsec_kprocess, sc);
547	#endif
548	}
549	}
550
551	static int
552	ubsec_detach(device_t self, int flags)
553	{
554	struct ubsec_softc *sc = device_private(self);
555	struct ubsec_q q, qtmp;
556	volatile u_int32_t ctrl;
557
558	/ disable interrupts /
559	/ XXX wait/abort current ops? where is DMAERR enabled? /
560	ctrl = READ_REG(sc, BS_CTRL);
561
562	ctrl &= ~(BS_CTRL_MCR2INT \| BS_CTRL_MCR1INT \| BS_CTRL_DMAERR);
563	if (sc->sc_flags & UBS_FLAGS_MULTIMCR)
564	ctrl &= ~BS_CTRL_MCR4INT;
565
566	WRITE_REG(sc, BS_CTRL, ctrl);
567
568	#ifndef UBSEC_NO_RNG
569	if (sc->sc_flags & UBS_FLAGS_RNG) {
570	callout_halt(&sc->sc_rngto, NULL);
571	ubsec_dma_free(sc, &sc->sc_rng.rng_buf);
572	ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx);
573	ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr);
574	rnd_detach_source(&sc->sc_rnd_source);
575	}
576	#endif /* UBSEC_NO_RNG */
577
578	crypto_unregister_all(sc->sc_cid);
579
580	mutex_spin_enter(&sc->sc_mtx);
581
582	ubsec_totalreset(sc); / XXX leaves the chip running /
583
584	SIMPLEQ_FOREACH_SAFE(q, &sc->sc_freequeue, q_next, qtmp) {
585	ubsec_dma_free(sc, &q->q_dma->d_alloc);
586	if (q->q_src_map != NULL)
587	bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
588	if (q->q_cached_dst_map != NULL)
589	bus_dmamap_destroy(sc->sc_dmat, q->q_cached_dst_map);
590	free(q, M_DEVBUF);
591	}
592
593	mutex_spin_exit(&sc->sc_mtx);
594
595	if (sc->sc_ih != NULL) {
596	pci_intr_disestablish(sc->sc_pct, sc->sc_ih);
597	sc->sc_ih = NULL;
598	}
599
600	if (sc->sc_memsize != `0`) {
601	bus_space_unmap(sc->sc_st, sc->sc_sh, sc->sc_memsize);
602	sc->sc_memsize = `0`;
603	}
604
605	return `0`;
606	}
607
608	MODULE(MODULE_CLASS_DRIVER, ubsec, "pci,opencrypto");
609
610	#ifdef _MODULE
611	#include "ioconf.c"
612	#endif
613
614	static int
615	ubsec_modcmd(modcmd_t cmd, void *data)
616	{
617	int error = `0`;
618
619	switch (cmd) {
620	case MODULE_CMD_INIT:
621	#ifdef _MODULE
622	error = config_init_component(cfdriver_ioconf_ubsec,
623	cfattach_ioconf_ubsec, cfdata_ioconf_ubsec);
624	#endif
625	if (error == `0`)
626	error = ubsec_sysctl_init();
627	return error;
628	case MODULE_CMD_FINI:
629	if (ubsec_sysctllog != NULL)
630	sysctl_teardown(&ubsec_sysctllog);
631	#ifdef _MODULE
632	error = config_fini_component(cfdriver_ioconf_ubsec,
633	cfattach_ioconf_ubsec, cfdata_ioconf_ubsec);
634	#endif
635	return error;
636	default:
637	return ENOTTY;
638	}
639	}
640
641	static int
642	ubsec_sysctl_init(void)
643	{
644	const struct sysctlnode *node = NULL;
645
646	ubsec_sysctllog = NULL;
647
648	sysctl_createv(&ubsec_sysctllog, `0`, NULL, &node,
649	CTLFLAG_PERMANENT,
650	CTLTYPE_NODE, "ubsec",
651	SYSCTL_DESCR("ubsec opetions"),
652	NULL, `0`, NULL, `0`,
653	CTL_HW, CTL_CREATE, CTL_EOL);
654	sysctl_createv(&ubsec_sysctllog, `0`, &node, NULL,
655	CTLFLAG_PERMANENT \| CTLFLAG_READWRITE,
656	CTLTYPE_INT, "maxbatch",
657	SYSCTL_DESCR("max ops to batch w/o interrupt"),
658	NULL, `0`, &ubsec_maxbatch, `0`,
659	CTL_CREATE, CTL_EOL);
660	sysctl_createv(&ubsec_sysctllog, `0`, &node, NULL,
661	CTLFLAG_PERMANENT \| CTLFLAG_READWRITE,
662	CTLTYPE_INT, "maxaggr",
663	SYSCTL_DESCR("max ops to aggregate under one interrupt"),
664	NULL, `0`, &ubsec_maxaggr, `0`,
665	CTL_CREATE, CTL_EOL);
666
667	return `0`;
668	}
669
670	/*
671	* UBSEC Interrupt routine
672	*/
673	static int
674	ubsec_intr(void *arg)
675	{
676	struct ubsec_softc *sc = arg;
677	volatile u_int32_t stat;
678	struct ubsec_q *q;
679	struct ubsec_dma *dmap;
680	int flags;
681	int npkts = `0`, i;
682
683	mutex_spin_enter(&sc->sc_mtx);
684	stat = READ_REG(sc, BS_STAT);
685	stat &= sc->sc_statmask;
686	if (stat == `0`) {
687	mutex_spin_exit(&sc->sc_mtx);
688	return (`0`);
689	}
690
691	WRITE_REG(sc, BS_STAT, stat); / IACK /
692
693	/*
694	* Check to see if we have any packets waiting for us
695	*/
696	if ((stat & BS_STAT_MCR1_DONE)) {
697	while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) {
698	q = SIMPLEQ_FIRST(&sc->sc_qchip);
699	dmap = q->q_dma;
700
701	if ((dmap->d_dma->d_mcr.mcr_flags
702	& htole16(UBS_MCR_DONE)) == `0`)
703	break;
704
705	q = SIMPLEQ_FIRST(&sc->sc_qchip);
706	SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, /q,/ q_next);
707
708	npkts = q->q_nstacked_mcrs;
709	sc->sc_nqchip -= `1`+npkts;
710	/*
711	* search for further sc_qchip ubsec_q's that share
712	* the same MCR, and complete them too, they must be
713	* at the top.
714	*/
715	for (i = `0`; i < npkts; i++) {
716	if(q->q_stacked_mcr[i])
717	ubsec_callback(sc, q->q_stacked_mcr[i]);
718	else
719	break;
720	}
721	ubsec_callback(sc, q);
722	}
723
724	/*
725	* Don't send any more packet to chip if there has been
726	* a DMAERR.
727	*/
728	if (!(stat & BS_STAT_DMAERR))
729	ubsec_feed(sc);
730	}
731
732	/*
733	* Check to see if we have any key setups/rng's waiting for us
734	*/
735	if ((sc->sc_flags & (UBS_FLAGS_KEY\|UBS_FLAGS_RNG)) &&
736	(stat & BS_STAT_MCR2_DONE)) {
737	struct ubsec_q2 *q2;
738	struct ubsec_mcr *mcr;
739
740	while (!SIMPLEQ_EMPTY(&sc->sc_qchip2)) {
741	q2 = SIMPLEQ_FIRST(&sc->sc_qchip2);
742
743	bus_dmamap_sync(sc->sc_dmat, q2->q_mcr.dma_map,
744	`0`, q2->q_mcr.dma_map->dm_mapsize,
745	BUS_DMASYNC_POSTREAD\|BUS_DMASYNC_POSTWRITE);
746
747	mcr = (struct ubsec_mcr *)q2->q_mcr.dma_vaddr;
748
749	/ A bug in new devices requires to swap this field /
750	if (sc->sc_flags & UBS_FLAGS_MULTIMCR)
751	flags = htole16(mcr->mcr_flags);
752	else
753	flags = mcr->mcr_flags;
754	if ((flags & htole16(UBS_MCR_DONE)) == `0`) {
755	bus_dmamap_sync(sc->sc_dmat,
756	q2->q_mcr.dma_map, `0`,
757	q2->q_mcr.dma_map->dm_mapsize,
758	BUS_DMASYNC_PREREAD\|BUS_DMASYNC_PREWRITE);
759	break;
760	}
761	q2 = SIMPLEQ_FIRST(&sc->sc_qchip2);
762	SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip2, /q2,/ q_next);
763	ubsec_callback2(sc, q2);
764	/*
765	* Don't send any more packet to chip if there has been
766	* a DMAERR.
767	*/
768	if (!(stat & BS_STAT_DMAERR))
769	ubsec_feed2(sc);
770	}
771	}
772	if ((sc->sc_flags & UBS_FLAGS_RNG4) && (stat & BS_STAT_MCR4_DONE)) {
773	struct ubsec_q2 *q2;
774	struct ubsec_mcr *mcr;
775
776	while (!SIMPLEQ_EMPTY(&sc->sc_qchip4)) {
777	q2 = SIMPLEQ_FIRST(&sc->sc_qchip4);
778
779	bus_dmamap_sync(sc->sc_dmat, q2->q_mcr.dma_map,
780	`0`, q2->q_mcr.dma_map->dm_mapsize,
781	BUS_DMASYNC_POSTREAD\|BUS_DMASYNC_POSTWRITE);
782
783	mcr = (struct ubsec_mcr *)q2->q_mcr.dma_vaddr;
784
785	/ A bug in new devices requires to swap this field /
786	flags = htole16(mcr->mcr_flags);
787
788	if ((flags & htole16(UBS_MCR_DONE)) == `0`) {
789	bus_dmamap_sync(sc->sc_dmat,
790	q2->q_mcr.dma_map, `0`,
791	q2->q_mcr.dma_map->dm_mapsize,
792	BUS_DMASYNC_PREREAD\|BUS_DMASYNC_PREWRITE);
793	break;
794	}
795	SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip4, q_next);
796	ubsec_callback2(sc, q2);
797	/*
798	* Don't send any more packet to chip if there has been
799	* a DMAERR.
800	*/
801	if (!(stat & BS_STAT_DMAERR))
802	ubsec_feed4(sc);
803	}
804	}
805
806	/*
807	* Check to see if we got any DMA Error
808	*/
809	if (stat & BS_STAT_DMAERR) {
810	#ifdef UBSEC_DEBUG
811	if (ubsec_debug) {
812	volatile u_int32_t a = READ_REG(sc, BS_ERR);
813
814	printf("%s: dmaerr %s@%08x\n", device_xname(sc->sc_dev),
815	(a & BS_ERR_READ) ? "read" : "write",
816	a & BS_ERR_ADDR);
817	}
818	#endif /* UBSEC_DEBUG */
819	ubsecstats.hst_dmaerr++;
820	ubsec_totalreset(sc);
821	ubsec_feed(sc);
822	}
823
824	if (sc->sc_needwakeup) { / XXX check high watermark /
825	int wkeup = sc->sc_needwakeup & (CRYPTO_SYMQ\|CRYPTO_ASYMQ);
826	#ifdef UBSEC_DEBUG
827	if (ubsec_debug)
828	printf("%s: wakeup crypto (%x)\n",
829	device_xname(sc->sc_dev), sc->sc_needwakeup);
830	#endif /* UBSEC_DEBUG */
831	sc->sc_needwakeup &= ~wkeup;
832	crypto_unblock(sc->sc_cid, wkeup);
833	}
834	mutex_spin_exit(&sc->sc_mtx);
835	return (`1`);
836	}
837
838	/*
839	* ubsec_feed() - aggregate and post requests to chip
840	* OpenBSD comments:
841	* It is assumed that the caller set splnet()
842	*/
843	static void
844	ubsec_feed(struct ubsec_softc *sc)
845	{
846	struct ubsec_q q, q2;
847	int npkts, i;
848	void *v;
849	u_int32_t stat;
850	#ifdef UBSEC_DEBUG
851	static int max;
852	#endif /* UBSEC_DEBUG */
853
854	npkts = sc->sc_nqueue;
855	if (npkts > ubsecstats.hst_maxqueue)
856	ubsecstats.hst_maxqueue = npkts;
857	if (npkts < `2`)
858	goto feed1;
859
860	/*
861	* Decide how many ops to combine in a single MCR. We cannot
862	* aggregate more than UBS_MAX_AGGR because this is the number
863	* of slots defined in the data structure. Otherwise we clamp
864	* based on the tunable parameter ubsec_maxaggr. Note that
865	* aggregation can happen in two ways: either by batching ops
866	* from above or because the h/w backs up and throttles us.
867	* Aggregating ops reduces the number of interrupts to the host
868	* but also (potentially) increases the latency for processing
869	* completed ops as we only get an interrupt when all aggregated
870	* ops have completed.
871	*/
872	if (npkts > sc->sc_maxaggr)
873	npkts = sc->sc_maxaggr;
874	if (npkts > ubsec_maxaggr)
875	npkts = ubsec_maxaggr;
876	if (npkts > ubsecstats.hst_maxbatch)
877	ubsecstats.hst_maxbatch = npkts;
878	if (npkts < `2`)
879	goto feed1;
880	ubsecstats.hst_totbatch += npkts-`1`;
881
882	if ((stat = READ_REG(sc, BS_STAT))
883	& (BS_STAT_MCR1_FULL \| BS_STAT_DMAERR)) {
884	if (stat & BS_STAT_DMAERR) {
885	ubsec_totalreset(sc);
886	ubsecstats.hst_dmaerr++;
887	} else {
888	ubsecstats.hst_mcr1full++;
889	}
890	return;
891	}
892
893	#ifdef UBSEC_DEBUG
894	if (ubsec_debug)
895	printf("merging %d records\n", npkts);
896	/ XXX temporary aggregation statistics reporting code /
897	if (max < npkts) {
898	max = npkts;
899	printf("%s: new max aggregate %d\n", device_xname(sc->sc_dev),
900	max);
901	}
902	#endif /* UBSEC_DEBUG */
903
904	q = SIMPLEQ_FIRST(&sc->sc_queue);
905	SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, /q,/ q_next);
906	--sc->sc_nqueue;
907
908	bus_dmamap_sync(sc->sc_dmat, q->q_src_map,
909	`0`, q->q_src_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
910	if (q->q_dst_map != NULL)
911	bus_dmamap_sync(sc->sc_dmat, q->q_dst_map,
912	`0`, q->q_dst_map->dm_mapsize, BUS_DMASYNC_PREREAD);
913
914	q->q_nstacked_mcrs = npkts - `1`; / Number of packets stacked /
915
916	for (i = `0`; i < q->q_nstacked_mcrs; i++) {
917	q2 = SIMPLEQ_FIRST(&sc->sc_queue);
918	bus_dmamap_sync(sc->sc_dmat, q2->q_src_map,
919	`0`, q2->q_src_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
920	if (q2->q_dst_map != NULL)
921	bus_dmamap_sync(sc->sc_dmat, q2->q_dst_map,
922	`0`, q2->q_dst_map->dm_mapsize, BUS_DMASYNC_PREREAD);
923	q2= SIMPLEQ_FIRST(&sc->sc_queue);
924	SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, /q2,/ q_next);
925	--sc->sc_nqueue;
926
927	v = ((void *)&q2->q_dma->d_dma->d_mcr);
928	v = (char)v + (sizeof(struct* ubsec_mcr) -
929	sizeof(struct ubsec_mcr_add));
930	memcpy(&q->q_dma->d_dma->d_mcradd[i], v,
931	sizeof(struct ubsec_mcr_add));
932	q->q_stacked_mcr[i] = q2;
933	}
934	q->q_dma->d_dma->d_mcr.mcr_pkts = htole16(npkts);
935	SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next);
936	sc->sc_nqchip += npkts;
937	if (sc->sc_nqchip > ubsecstats.hst_maxqchip)
938	ubsecstats.hst_maxqchip = sc->sc_nqchip;
939	bus_dmamap_sync(sc->sc_dmat, q->q_dma->d_alloc.dma_map,
940	`0`, q->q_dma->d_alloc.dma_map->dm_mapsize,
941	BUS_DMASYNC_PREREAD \| BUS_DMASYNC_PREWRITE);
942	WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr +
943	offsetof(struct ubsec_dmachunk, d_mcr));
944	return;
945
946	feed1:
947	while (!SIMPLEQ_EMPTY(&sc->sc_queue)) {
948	if ((stat = READ_REG(sc, BS_STAT))
949	& (BS_STAT_MCR1_FULL \| BS_STAT_DMAERR)) {
950	if (stat & BS_STAT_DMAERR) {
951	ubsec_totalreset(sc);
952	ubsecstats.hst_dmaerr++;
953	} else {
954	ubsecstats.hst_mcr1full++;
955	}
956	break;
957	}
958
959	q = SIMPLEQ_FIRST(&sc->sc_queue);
960
961	bus_dmamap_sync(sc->sc_dmat, q->q_src_map,
962	`0`, q->q_src_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
963	if (q->q_dst_map != NULL)
964	bus_dmamap_sync(sc->sc_dmat, q->q_dst_map,
965	`0`, q->q_dst_map->dm_mapsize, BUS_DMASYNC_PREREAD);
966	bus_dmamap_sync(sc->sc_dmat, q->q_dma->d_alloc.dma_map,
967	`0`, q->q_dma->d_alloc.dma_map->dm_mapsize,
968	BUS_DMASYNC_PREREAD \| BUS_DMASYNC_PREWRITE);
969
970	WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr +
971	offsetof(struct ubsec_dmachunk, d_mcr));
972	#ifdef UBSEC_DEBUG
973	if (ubsec_debug)
974	printf("feed: q->chip %p %08x stat %08x\n",
975	q, (u_int32_t)q->q_dma->d_alloc.dma_paddr,
976	stat);
977	#endif /* UBSEC_DEBUG */
978	q = SIMPLEQ_FIRST(&sc->sc_queue);
979	SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, /q,/ q_next);
980	--sc->sc_nqueue;
981	SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next);
982	sc->sc_nqchip++;
983	}
984	if (sc->sc_nqchip > ubsecstats.hst_maxqchip)
985	ubsecstats.hst_maxqchip = sc->sc_nqchip;
986	}
987
988	/*
989	* Allocate a new 'session' and return an encoded session id. 'sidp'
990	* contains our registration id, and should contain an encoded session
991	* id on successful allocation.
992	*/
993	static int
994	ubsec_newsession(void arg, u_int32_t sidp, struct cryptoini *cri)
995	{
996	struct cryptoini c, encini = NULL, *macini = NULL;
997	struct ubsec_softc *sc;
998	struct ubsec_session *ses = NULL;
999	MD5_CTX md5ctx;
1000	SHA1_CTX sha1ctx;
1001	int i, sesn;
1002
1003	sc = arg;
1004	KASSERT(sc != NULL /, ("ubsec_newsession: null softc")/);
1005
1006	if (sidp == NULL \|\| cri == NULL \|\| sc == NULL)
1007	return (EINVAL);
1008
1009	for (c = cri; c != NULL; c = c->cri_next) {
1010	if (c->cri_alg == CRYPTO_MD5_HMAC_96 \|\|
1011	c->cri_alg == CRYPTO_SHA1_HMAC_96) {
1012	if (macini)
1013	return (EINVAL);
1014	macini = c;
1015	} else if (c->cri_alg == CRYPTO_DES_CBC \|\|
1016	c->cri_alg == CRYPTO_3DES_CBC \|\|
1017	c->cri_alg == CRYPTO_AES_CBC) {
1018	if (encini)
1019	return (EINVAL);
1020	encini = c;
1021	} else
1022	return (EINVAL);
1023	}
1024	if (encini == NULL && macini == NULL)
1025	return (EINVAL);
1026
1027	if (encini && encini->cri_alg == CRYPTO_AES_CBC) {
1028	switch (encini->cri_klen) {
1029	case `128`:
1030	case `192`:
1031	case `256`:
1032	break;
1033	default:
1034	return (EINVAL);
1035	}
1036	}
1037
1038	if (sc->sc_sessions == NULL) {
1039	ses = sc->sc_sessions = (struct ubsec_session *)malloc(
1040	sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT);
1041	if (ses == NULL)
1042	return (ENOMEM);
1043	sesn = `0`;
1044	sc->sc_nsessions = `1`;
1045	} else {
1046	for (sesn = `0`; sesn < sc->sc_nsessions; sesn++) {
1047	if (sc->sc_sessions[sesn].ses_used == `0`) {
1048	ses = &sc->sc_sessions[sesn];
1049	break;
1050	}
1051	}
1052
1053	if (ses == NULL) {
1054	sesn = sc->sc_nsessions;
1055	ses = (struct ubsec_session )malloc((sesn + `1`)
1056	sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT);
1057	if (ses == NULL)
1058	return (ENOMEM);
1059	memcpy(ses, sc->sc_sessions, sesn *
1060	sizeof(struct ubsec_session));
1061	memset(sc->sc_sessions, `0`, sesn *
1062	sizeof(struct ubsec_session));
1063	free(sc->sc_sessions, M_DEVBUF);
1064	sc->sc_sessions = ses;
1065	ses = &sc->sc_sessions[sesn];
1066	sc->sc_nsessions++;
1067	}
1068	}
1069
1070	memset(ses, `0`, sizeof(struct ubsec_session));
1071	ses->ses_used = `1`;
1072	if (encini) {
1073	/ get an IV, network byte order /
1074	#ifdef __NetBSD__
1075	cprng_fast(ses->ses_iv, sizeof(ses->ses_iv));
1076	#else
1077	get_random_bytes(ses->ses_iv, sizeof(ses->ses_iv));
1078	#endif
1079
1080	/ Go ahead and compute key in ubsec's byte order /
1081	if (encini->cri_alg == CRYPTO_AES_CBC) {
1082	memcpy(ses->ses_key, encini->cri_key,
1083	encini->cri_klen / `8`);
1084	}
1085	if (encini->cri_alg == CRYPTO_DES_CBC) {
1086	memcpy(&ses->ses_key[`0`], encini->cri_key, `8`);
1087	memcpy(&ses->ses_key[`2`], encini->cri_key, `8`);
1088	memcpy(&ses->ses_key[`4`], encini->cri_key, `8`);
1089	} else
1090	memcpy(ses->ses_key, encini->cri_key, `24`);
1091
1092	SWAP32(ses->ses_key[`0`]);
1093	SWAP32(ses->ses_key[`1`]);
1094	SWAP32(ses->ses_key[`2`]);
1095	SWAP32(ses->ses_key[`3`]);
1096	SWAP32(ses->ses_key[`4`]);
1097	SWAP32(ses->ses_key[`5`]);
1098	}
1099
1100	if (macini) {
1101	for (i = `0`; i < macini->cri_klen / `8`; i++)
1102	macini->cri_key[i] ^= HMAC_IPAD_VAL;
1103
1104	if (macini->cri_alg == CRYPTO_MD5_HMAC_96) {
1105	MD5Init(&md5ctx);
1106	MD5Update(&md5ctx, macini->cri_key,
1107	macini->cri_klen / `8`);
1108	MD5Update(&md5ctx, hmac_ipad_buffer,
1109	HMAC_BLOCK_LEN - (macini->cri_klen / `8`));
1110	memcpy(ses->ses_hminner, md5ctx.state,
1111	sizeof(md5ctx.state));
1112	} else {
1113	SHA1Init(&sha1ctx);
1114	SHA1Update(&sha1ctx, macini->cri_key,
1115	macini->cri_klen / `8`);
1116	SHA1Update(&sha1ctx, hmac_ipad_buffer,
1117	HMAC_BLOCK_LEN - (macini->cri_klen / `8`));
1118	memcpy(ses->ses_hminner, sha1ctx.state,
1119	sizeof(sha1ctx.state));
1120	}
1121
1122	for (i = `0`; i < macini->cri_klen / `8`; i++)
1123	macini->cri_key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL);
1124
1125	if (macini->cri_alg == CRYPTO_MD5_HMAC_96) {
1126	MD5Init(&md5ctx);
1127	MD5Update(&md5ctx, macini->cri_key,
1128	macini->cri_klen / `8`);
1129	MD5Update(&md5ctx, hmac_opad_buffer,
1130	HMAC_BLOCK_LEN - (macini->cri_klen / `8`));
1131	memcpy(ses->ses_hmouter, md5ctx.state,
1132	sizeof(md5ctx.state));
1133	} else {
1134	SHA1Init(&sha1ctx);
1135	SHA1Update(&sha1ctx, macini->cri_key,
1136	macini->cri_klen / `8`);
1137	SHA1Update(&sha1ctx, hmac_opad_buffer,
1138	HMAC_BLOCK_LEN - (macini->cri_klen / `8`));
1139	memcpy(ses->ses_hmouter, sha1ctx.state,
1140	sizeof(sha1ctx.state));
1141	}
1142
1143	for (i = `0`; i < macini->cri_klen / `8`; i++)
1144	macini->cri_key[i] ^= HMAC_OPAD_VAL;
1145	}
1146
1147	*sidp = UBSEC_SID(device_unit(sc->sc_dev), sesn);
1148	return (`0`);
1149	}
1150
1151	/*
1152	* Deallocate a session.
1153	*/
1154	static int
1155	ubsec_freesession(void *arg, u_int64_t tid)
1156	{
1157	struct ubsec_softc *sc;
1158	int session;
1159	u_int32_t sid = ((u_int32_t) tid) & `0xffffffff`;
1160
1161	sc = arg;
1162	KASSERT(sc != NULL /, ("ubsec_freesession: null softc")/);
1163
1164	session = UBSEC_SESSION(sid);
1165	if (session >= sc->sc_nsessions)
1166	return (EINVAL);
1167
1168	memset(&sc->sc_sessions[session], `0`, sizeof(sc->sc_sessions[session]));
1169	return (`0`);
1170	}
1171
1172	#ifdef __FreeBSD__ /* Ugly gratuitous changes to bus_dma */
1173	static void
1174	ubsec_op_cb(void arg, bus_dma_segment_t seg, int nsegs, bus_size_t mapsize,
1175	int error)
1176	{
1177	struct ubsec_operand *op = arg;
1178
1179	KASSERT(nsegs <= UBS_MAX_SCATTER
1180	/, ("Too many DMA segments returned when mapping operand")/);
1181	#ifdef UBSEC_DEBUG
1182	if (ubsec_debug)
1183	printf("ubsec_op_cb: mapsize %u nsegs %d\n",
1184	(u_int) mapsize, nsegs);
1185	#endif
1186	op->mapsize = mapsize;
1187	op->nsegs = nsegs;
1188	memcpy(op->segs, seg, nsegs * sizeof (seg[`0`]));
1189	}
1190	#endif
1191
1192	static int
1193	ubsec_process(void arg, struct* cryptop crp, int* hint)
1194	{
1195	struct ubsec_q *q = NULL;
1196	#ifdef __OpenBSD__
1197	int card;
1198	#endif
1199	int err = `0`, i, j, nicealign;
1200	struct ubsec_softc *sc;
1201	struct cryptodesc crd1, crd2, maccrd, enccrd;
1202	int encoffset = `0`, macoffset = `0`, cpskip, cpoffset;
1203	int sskip, dskip, stheend, dtheend;
1204	int16_t coffset;
1205	struct ubsec_session *ses, key;
1206	struct ubsec_dma *dmap = NULL;
1207	u_int16_t flags = `0`;
1208	int ivlen = `0`, keylen = `0`;
1209
1210	sc = arg;
1211	KASSERT(sc != NULL /, ("ubsec_process: null softc")/);
1212
1213	if (crp == NULL \|\| crp->crp_callback == NULL \|\| sc == NULL) {
1214	ubsecstats.hst_invalid++;
1215	return (EINVAL);
1216	}
1217	if (UBSEC_SESSION(crp->crp_sid) >= sc->sc_nsessions) {
1218	ubsecstats.hst_badsession++;
1219	return (EINVAL);
1220	}
1221
1222	mutex_spin_enter(&sc->sc_mtx);
1223
1224	if (SIMPLEQ_EMPTY(&sc->sc_freequeue)) {
1225	ubsecstats.hst_queuefull++;
1226	sc->sc_needwakeup \|= CRYPTO_SYMQ;
1227	mutex_spin_exit(&sc->sc_mtx);
1228	return(ERESTART);
1229	}
1230
1231	q = SIMPLEQ_FIRST(&sc->sc_freequeue);
1232	SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, /q,/ q_next);
1233	mutex_spin_exit(&sc->sc_mtx);
1234
1235	dmap = q->q_dma; / Save dma pointer /
1236	/ don't lose the cached dmamaps q_src_map and q_cached_dst_map /
1237	memset(q, `0`, offsetof(struct ubsec_q, q_src_map));
1238	memset(&key, `0`, sizeof(key));
1239
1240	q->q_sesn = UBSEC_SESSION(crp->crp_sid);
1241	q->q_dma = dmap;
1242	ses = &sc->sc_sessions[q->q_sesn];
1243
1244	if (crp->crp_flags & CRYPTO_F_IMBUF) {
1245	q->q_src_m = (struct mbuf *)crp->crp_buf;
1246	q->q_dst_m = (struct mbuf *)crp->crp_buf;
1247	} else if (crp->crp_flags & CRYPTO_F_IOV) {
1248	q->q_src_io = (struct uio *)crp->crp_buf;
1249	q->q_dst_io = (struct uio *)crp->crp_buf;
1250	} else {
1251	ubsecstats.hst_badflags++;
1252	err = EINVAL;
1253	goto errout; / XXX we don't handle contiguous blocks! /
1254	}
1255
1256	memset(&dmap->d_dma->d_mcr, `0`, sizeof(struct ubsec_mcr));
1257
1258	dmap->d_dma->d_mcr.mcr_pkts = htole16(`1`);
1259	dmap->d_dma->d_mcr.mcr_flags = `0`;
1260	q->q_crp = crp;
1261
1262	crd1 = crp->crp_desc;
1263	if (crd1 == NULL) {
1264	ubsecstats.hst_nodesc++;
1265	err = EINVAL;
1266	goto errout;
1267	}
1268	crd2 = crd1->crd_next;
1269
1270	if (crd2 == NULL) {
1271	if (crd1->crd_alg == CRYPTO_MD5_HMAC_96 \|\|
1272	crd1->crd_alg == CRYPTO_SHA1_HMAC_96) {
1273	maccrd = crd1;
1274	enccrd = NULL;
1275	} else if (crd1->crd_alg == CRYPTO_DES_CBC \|\|
1276	crd1->crd_alg == CRYPTO_3DES_CBC \|\|
1277	crd1->crd_alg == CRYPTO_AES_CBC) {
1278	maccrd = NULL;
1279	enccrd = crd1;
1280	} else {
1281	ubsecstats.hst_badalg++;
1282	err = EINVAL;
1283	goto errout;
1284	}
1285	} else {
1286	if ((crd1->crd_alg == CRYPTO_MD5_HMAC_96 \|\|
1287	crd1->crd_alg == CRYPTO_SHA1_HMAC_96) &&
1288	(crd2->crd_alg == CRYPTO_DES_CBC \|\|
1289	crd2->crd_alg == CRYPTO_3DES_CBC \|\|
1290	crd2->crd_alg == CRYPTO_AES_CBC) &&
1291	((crd2->crd_flags & CRD_F_ENCRYPT) == `0`)) {
1292	maccrd = crd1;
1293	enccrd = crd2;
1294	} else if ((crd1->crd_alg == CRYPTO_DES_CBC \|\|
1295	crd1->crd_alg == CRYPTO_3DES_CBC \|\|
1296	crd1->crd_alg == CRYPTO_AES_CBC) &&
1297	(crd2->crd_alg == CRYPTO_MD5_HMAC_96 \|\|
1298	crd2->crd_alg == CRYPTO_SHA1_HMAC_96) &&
1299	(crd1->crd_flags & CRD_F_ENCRYPT)) {
1300	enccrd = crd1;
1301	maccrd = crd2;
1302	} else {
1303	/*
1304	* We cannot order the ubsec as requested
1305	*/
1306	ubsecstats.hst_badalg++;
1307	err = EINVAL;
1308	goto errout;
1309	}
1310	}
1311
1312	if (enccrd) {
1313	if (enccrd->crd_alg == CRYPTO_AES_CBC) {
1314	if ((sc->sc_flags & UBS_FLAGS_AES) == `0`) {
1315	/*
1316	* We cannot order the ubsec as requested
1317	*/
1318	ubsecstats.hst_badalg++;
1319	err = EINVAL;
1320	goto errout;
1321	}
1322	flags \|= htole16(UBS_PKTCTX_ENC_AES);
1323	switch (enccrd->crd_klen) {
1324	case `128`:
1325	case `192`:
1326	case `256`:
1327	keylen = enccrd->crd_klen / `8`;
1328	break;
1329	default:
1330	err = EINVAL;
1331	goto errout;
1332	}
1333	ivlen = `16`;
1334	} else {
1335	flags \|= htole16(UBS_PKTCTX_ENC_3DES);
1336	ivlen = `8`;
1337	keylen = `24`;
1338	}
1339
1340	encoffset = enccrd->crd_skip;
1341
1342	if (enccrd->crd_flags & CRD_F_ENCRYPT) {
1343	q->q_flags \|= UBSEC_QFLAGS_COPYOUTIV;
1344
1345	if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
1346	memcpy(key.ses_iv, enccrd->crd_iv, ivlen);
1347	else {
1348	for (i = `0`; i < (ivlen / `4`); i++)
1349	key.ses_iv[i] = ses->ses_iv[i];
1350	}
1351
1352	if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == `0`) {
1353	if (crp->crp_flags & CRYPTO_F_IMBUF)
1354	m_copyback(q->q_src_m,
1355	enccrd->crd_inject,
1356	ivlen, (void *)key.ses_iv);
1357	else if (crp->crp_flags & CRYPTO_F_IOV)
1358	cuio_copyback(q->q_src_io,
1359	enccrd->crd_inject,
1360	ivlen, (void *)key.ses_iv);
1361	}
1362	} else {
1363	flags \|= htole16(UBS_PKTCTX_INBOUND);
1364
1365	if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
1366	memcpy(key.ses_iv, enccrd->crd_iv, ivlen);
1367	else if (crp->crp_flags & CRYPTO_F_IMBUF)
1368	m_copydata(q->q_src_m, enccrd->crd_inject,
1369	ivlen, (void *)key.ses_iv);
1370	else if (crp->crp_flags & CRYPTO_F_IOV)
1371	cuio_copydata(q->q_src_io,
1372	enccrd->crd_inject, `8`,
1373	(void *)key.ses_iv);
1374	}
1375
1376	for (i = `0`; i < (keylen / `4`); i++)
1377	key.ses_key[i] = ses->ses_key[i];
1378	for (i = `0`; i < (ivlen / `4`); i++)
1379	SWAP32(key.ses_iv[i]);
1380	}
1381
1382	if (maccrd) {
1383	macoffset = maccrd->crd_skip;
1384
1385	if (maccrd->crd_alg == CRYPTO_MD5_HMAC_96)
1386	flags \|= htole16(UBS_PKTCTX_AUTH_MD5);
1387	else
1388	flags \|= htole16(UBS_PKTCTX_AUTH_SHA1);
1389
1390	for (i = `0`; i < `5`; i++) {
1391	key.ses_hminner[i] = ses->ses_hminner[i];
1392	key.ses_hmouter[i] = ses->ses_hmouter[i];
1393
1394	HTOLE32(key.ses_hminner[i]);
1395	HTOLE32(key.ses_hmouter[i]);
1396	}
1397	}
1398
1399	if (enccrd && maccrd) {
1400	/*
1401	* ubsec cannot handle packets where the end of encryption
1402	* and authentication are not the same, or where the
1403	* encrypted part begins before the authenticated part.
1404	*/
1405	if ((encoffset + enccrd->crd_len) !=
1406	(macoffset + maccrd->crd_len)) {
1407	ubsecstats.hst_lenmismatch++;
1408	err = EINVAL;
1409	goto errout;
1410	}
1411	if (enccrd->crd_skip < maccrd->crd_skip) {
1412	ubsecstats.hst_skipmismatch++;
1413	err = EINVAL;
1414	goto errout;
1415	}
1416	sskip = maccrd->crd_skip;
1417	cpskip = dskip = enccrd->crd_skip;
1418	stheend = maccrd->crd_len;
1419	dtheend = enccrd->crd_len;
1420	coffset = enccrd->crd_skip - maccrd->crd_skip;
1421	cpoffset = cpskip + dtheend;
1422	#ifdef UBSEC_DEBUG
1423	if (ubsec_debug) {
1424	printf("mac: skip %d, len %d, inject %d\n",
1425	maccrd->crd_skip, maccrd->crd_len,
1426	maccrd->crd_inject);
1427	printf("enc: skip %d, len %d, inject %d\n",
1428	enccrd->crd_skip, enccrd->crd_len,
1429	enccrd->crd_inject);
1430	printf("src: skip %d, len %d\n", sskip, stheend);
1431	printf("dst: skip %d, len %d\n", dskip, dtheend);
1432	printf("ubs: coffset %d, pktlen %d, cpskip %d, cpoffset %d\n",
1433	coffset, stheend, cpskip, cpoffset);
1434	}
1435	#endif
1436	} else {
1437	cpskip = dskip = sskip = macoffset + encoffset;
1438	dtheend = stheend = (enccrd)?enccrd->crd_len:maccrd->crd_len;
1439	cpoffset = cpskip + dtheend;
1440	coffset = `0`;
1441	}
1442
1443	if (q->q_src_map == NULL) {
1444	/ XXX FIXME: jonathan asks, what the heck's that 0xfff0? /
1445	if (bus_dmamap_create(sc->sc_dmat, `0xfff0`, UBS_MAX_SCATTER,
1446	`0xfff0`, `0`, BUS_DMA_NOWAIT, &q->q_src_map) != `0`) {
1447	err = ENOMEM;
1448	goto errout;
1449	}
1450	}
1451	if (crp->crp_flags & CRYPTO_F_IMBUF) {
1452	if (bus_dmamap_load_mbuf(sc->sc_dmat, q->q_src_map,
1453	q->q_src_m, BUS_DMA_NOWAIT) != `0`) {
1454	ubsecstats.hst_noload++;
1455	err = ENOMEM;
1456	goto errout;
1457	}
1458	} else if (crp->crp_flags & CRYPTO_F_IOV) {
1459	if (bus_dmamap_load_uio(sc->sc_dmat, q->q_src_map,
1460	q->q_src_io, BUS_DMA_NOWAIT) != `0`) {
1461	ubsecstats.hst_noload++;
1462	err = ENOMEM;
1463	goto errout;
1464	}
1465	}
1466	nicealign = ubsec_dmamap_aligned(q->q_src_map);
1467
1468	dmap->d_dma->d_mcr.mcr_pktlen = htole16(stheend);
1469
1470	#ifdef UBSEC_DEBUG
1471	if (ubsec_debug)
1472	printf("src skip: %d nicealign: %u\n", sskip, nicealign);
1473	#endif
1474	for (i = j = `0`; i < q->q_src_map->dm_nsegs; i++) {
1475	struct ubsec_pktbuf *pb;
1476	bus_size_t packl = q->q_src_map->dm_segs[i].ds_len;
1477	bus_addr_t packp = q->q_src_map->dm_segs[i].ds_addr;
1478
1479	if (sskip >= packl) {
1480	sskip -= packl;
1481	continue;
1482	}
1483
1484	packl -= sskip;
1485	packp += sskip;
1486	sskip = `0`;
1487
1488	if (packl > `0xfffc`) {
1489	err = EIO;
1490	goto errout;
1491	}
1492
1493	if (j == `0`)
1494	pb = &dmap->d_dma->d_mcr.mcr_ipktbuf;
1495	else
1496	pb = &dmap->d_dma->d_sbuf[j - `1`];
1497
1498	pb->pb_addr = htole32(packp);
1499
1500	if (stheend) {
1501	if (packl > stheend) {
1502	pb->pb_len = htole32(stheend);
1503	stheend = `0`;
1504	} else {
1505	pb->pb_len = htole32(packl);
1506	stheend -= packl;
1507	}
1508	} else
1509	pb->pb_len = htole32(packl);
1510
1511	if ((i + `1`) == q->q_src_map->dm_nsegs)
1512	pb->pb_next = `0`;
1513	else
1514	pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
1515	offsetof(struct ubsec_dmachunk, d_sbuf[j]));
1516	j++;
1517	}
1518
1519	if (enccrd == NULL && maccrd != NULL) {
1520	dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr = `0`;
1521	dmap->d_dma->d_mcr.mcr_opktbuf.pb_len = `0`;
1522	dmap->d_dma->d_mcr.mcr_opktbuf.pb_next = htole32(dmap->d_alloc.dma_paddr +
1523	offsetof(struct ubsec_dmachunk, d_macbuf[`0`]));
1524	#ifdef UBSEC_DEBUG
1525	if (ubsec_debug)
1526	printf("opkt: %x %x %x\n",
1527	dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr,
1528	dmap->d_dma->d_mcr.mcr_opktbuf.pb_len,
1529	dmap->d_dma->d_mcr.mcr_opktbuf.pb_next);
1530
1531	#endif
1532	} else {
1533	if (crp->crp_flags & CRYPTO_F_IOV) {
1534	if (!nicealign) {
1535	ubsecstats.hst_iovmisaligned++;
1536	err = EINVAL;
1537	goto errout;
1538	}
1539	if (q->q_dst_map == NULL) {
1540	if (q->q_cached_dst_map == NULL) {
1541	/*
1542	* XXX: ``what the heck's that''
1543	* 0xfff0?
1544	*/
1545	if (bus_dmamap_create(sc->sc_dmat,
1546	`0xfff0`, UBS_MAX_SCATTER, `0xfff0`, `0`,
1547	BUS_DMA_NOWAIT,
1548	&q->q_cached_dst_map) != `0`) {
1549	ubsecstats.hst_nomap++;
1550	err = ENOMEM;
1551	goto errout;
1552	}
1553	}
1554	q->q_dst_map = q->q_cached_dst_map;
1555	}
1556	if (bus_dmamap_load_uio(sc->sc_dmat, q->q_dst_map,
1557	q->q_dst_io, BUS_DMA_NOWAIT) != `0`) {
1558	ubsecstats.hst_noload++;
1559	err = ENOMEM;
1560	goto errout;
1561	}
1562	} else if (crp->crp_flags & CRYPTO_F_IMBUF) {
1563	if (nicealign) {
1564	q->q_dst_m = q->q_src_m;
1565	q->q_dst_map = q->q_src_map;
1566	} else {
1567	int totlen, len;
1568	struct mbuf m, top, **mp;
1569
1570	ubsecstats.hst_unaligned++;
1571	totlen = q->q_src_map->dm_mapsize;
1572	if (q->q_src_m->m_flags & M_PKTHDR) {
1573	len = MHLEN;
1574	MGETHDR(m, M_DONTWAIT, MT_DATA);
1575	/XXX FIXME: m_dup_pkthdr /
1576	if (m && `1` /!m_dup_pkthdr(m, q->q_src_m, M_DONTWAIT)/) {
1577	m_free(m);
1578	m = NULL;
1579	}
1580	} else {
1581	len = MLEN;
1582	MGET(m, M_DONTWAIT, MT_DATA);
1583	}
1584	if (m == NULL) {
1585	ubsecstats.hst_nombuf++;
1586	err = sc->sc_nqueue ? ERESTART : ENOMEM;
1587	goto errout;
1588	}
1589	if (len == MHLEN)
1590	/XXX was M_DUP_PKTHDR/
1591	M_COPY_PKTHDR(m, q->q_src_m);
1592	if (totlen >= MINCLSIZE) {
1593	MCLGET(m, M_DONTWAIT);
1594	if ((m->m_flags & M_EXT) == `0`) {
1595	m_free(m);
1596	ubsecstats.hst_nomcl++;
1597	err = sc->sc_nqueue
1598	? ERESTART : ENOMEM;
1599	goto errout;
1600	}
1601	len = MCLBYTES;
1602	}
1603	m->m_len = len;
1604	top = NULL;
1605	mp = &top;
1606
1607	while (totlen > `0`) {
1608	if (top) {
1609	MGET(m, M_DONTWAIT, MT_DATA);
1610	if (m == NULL) {
1611	m_freem(top);
1612	ubsecstats.hst_nombuf++;
1613	err = sc->sc_nqueue ? ERESTART : ENOMEM;
1614	goto errout;
1615	}
1616	len = MLEN;
1617	}
1618	if (top && totlen >= MINCLSIZE) {
1619	MCLGET(m, M_DONTWAIT);
1620	if ((m->m_flags & M_EXT) == `0`) {
1621	*mp = m;
1622	m_freem(top);
1623	ubsecstats.hst_nomcl++;
1624	err = sc->sc_nqueue ? ERESTART : ENOMEM;
1625	goto errout;
1626	}
1627	len = MCLBYTES;
1628	}
1629	m->m_len = len = min(totlen, len);
1630	totlen -= len;
1631	*mp = m;
1632	mp = &m->m_next;
1633	}
1634	q->q_dst_m = top;
1635	ubsec_mcopy(q->q_src_m, q->q_dst_m,
1636	cpskip, cpoffset);
1637	if (q->q_dst_map == NULL) {
1638	if (q->q_cached_dst_map == NULL) {
1639	/ XXX again, what the heck is that 0xfff0? /
1640	if (bus_dmamap_create(sc->sc_dmat, `0xfff0`,
1641	UBS_MAX_SCATTER, `0xfff0`, `0`, BUS_DMA_NOWAIT,
1642	&q->q_cached_dst_map) != `0`) {
1643	ubsecstats.hst_nomap++;
1644	err = ENOMEM;
1645	goto errout;
1646	}
1647	}
1648	q->q_dst_map = q->q_cached_dst_map;
1649	}
1650	if (bus_dmamap_load_mbuf(sc->sc_dmat,
1651	q->q_dst_map, q->q_dst_m,
1652	BUS_DMA_NOWAIT) != `0`) {
1653	ubsecstats.hst_noload++;
1654	err = ENOMEM;
1655	goto errout;
1656	}
1657	}
1658	} else {
1659	ubsecstats.hst_badflags++;
1660	err = EINVAL;
1661	goto errout;
1662	}
1663
1664	#ifdef UBSEC_DEBUG
1665	if (ubsec_debug)
1666	printf("dst skip: %d\n", dskip);
1667	#endif
1668	for (i = j = `0`; i < q->q_dst_map->dm_nsegs; i++) {
1669	struct ubsec_pktbuf *pb;
1670	bus_size_t packl = q->q_dst_map->dm_segs[i].ds_len;
1671	bus_addr_t packp = q->q_dst_map->dm_segs[i].ds_addr;
1672
1673	if (dskip >= packl) {
1674	dskip -= packl;
1675	continue;
1676	}
1677
1678	packl -= dskip;
1679	packp += dskip;
1680	dskip = `0`;
1681
1682	if (packl > `0xfffc`) {
1683	err = EIO;
1684	goto errout;
1685	}
1686
1687	if (j == `0`)
1688	pb = &dmap->d_dma->d_mcr.mcr_opktbuf;
1689	else
1690	pb = &dmap->d_dma->d_dbuf[j - `1`];
1691
1692	pb->pb_addr = htole32(packp);
1693
1694	if (dtheend) {
1695	if (packl > dtheend) {
1696	pb->pb_len = htole32(dtheend);
1697	dtheend = `0`;
1698	} else {
1699	pb->pb_len = htole32(packl);
1700	dtheend -= packl;
1701	}
1702	} else
1703	pb->pb_len = htole32(packl);
1704
1705	if ((i + `1`) == q->q_dst_map->dm_nsegs) {
1706	if (maccrd)
1707	pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
1708	offsetof(struct ubsec_dmachunk, d_macbuf[`0`]));
1709	else
1710	pb->pb_next = `0`;
1711	} else
1712	pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
1713	offsetof(struct ubsec_dmachunk, d_dbuf[j]));
1714	j++;
1715	}
1716	}
1717
1718	dmap->d_dma->d_mcr.mcr_cmdctxp = htole32(dmap->d_alloc.dma_paddr +
1719	offsetof(struct ubsec_dmachunk, d_ctx));
1720
1721	if (enccrd && enccrd->crd_alg == CRYPTO_AES_CBC) {
1722	struct ubsec_pktctx_aes128 *aes128;
1723	struct ubsec_pktctx_aes192 *aes192;
1724	struct ubsec_pktctx_aes256 *aes256;
1725	struct ubsec_pktctx_hdr *ph;
1726	u_int8_t *ctx;
1727
1728	ctx = (u_int8_t *)(dmap->d_alloc.dma_vaddr) +
1729	offsetof(struct ubsec_dmachunk, d_ctx);
1730
1731	ph = (struct ubsec_pktctx_hdr *)ctx;
1732	ph->ph_type = htole16(UBS_PKTCTX_TYPE_IPSEC_AES);
1733	ph->ph_flags = flags;
1734	ph->ph_offset = htole16(coffset >> `2`);
1735
1736	switch (enccrd->crd_klen) {
1737	case `128`:
1738	aes128 = (struct ubsec_pktctx_aes128 *)ctx;
1739	ph->ph_len = htole16(sizeof(*aes128));
1740	ph->ph_flags \|= htole16(UBS_PKTCTX_KEYSIZE_128);
1741	for (i = `0`; i < `4`; i++)
1742	aes128->pc_aeskey[i] = key.ses_key[i];
1743	for (i = `0`; i < `5`; i++)
1744	aes128->pc_hminner[i] = key.ses_hminner[i];
1745	for (i = `0`; i < `5`; i++)
1746	aes128->pc_hmouter[i] = key.ses_hmouter[i];
1747	for (i = `0`; i < `4`; i++)
1748	aes128->pc_iv[i] = key.ses_iv[i];
1749	break;
1750	case `192`:
1751	aes192 = (struct ubsec_pktctx_aes192 *)ctx;
1752	ph->ph_len = htole16(sizeof(*aes192));
1753	ph->ph_flags \|= htole16(UBS_PKTCTX_KEYSIZE_192);
1754	for (i = `0`; i < `6`; i++)
1755	aes192->pc_aeskey[i] = key.ses_key[i];
1756	for (i = `0`; i < `5`; i++)
1757	aes192->pc_hminner[i] = key.ses_hminner[i];
1758	for (i = `0`; i < `5`; i++)
1759	aes192->pc_hmouter[i] = key.ses_hmouter[i];
1760	for (i = `0`; i < `4`; i++)
1761	aes192->pc_iv[i] = key.ses_iv[i];
1762	break;
1763	case `256`:
1764	aes256 = (struct ubsec_pktctx_aes256 *)ctx;
1765	ph->ph_len = htole16(sizeof(*aes256));
1766	ph->ph_flags \|= htole16(UBS_PKTCTX_KEYSIZE_256);
1767	for (i = `0`; i < `8`; i++)
1768	aes256->pc_aeskey[i] = key.ses_key[i];
1769	for (i = `0`; i < `5`; i++)
1770	aes256->pc_hminner[i] = key.ses_hminner[i];
1771	for (i = `0`; i < `5`; i++)
1772	aes256->pc_hmouter[i] = key.ses_hmouter[i];
1773	for (i = `0`; i < `4`; i++)
1774	aes256->pc_iv[i] = key.ses_iv[i];
1775	break;
1776	}
1777	} else if (sc->sc_flags & UBS_FLAGS_LONGCTX) {
1778	struct ubsec_pktctx_3des *ctx;
1779	struct ubsec_pktctx_hdr *ph;
1780
1781	ctx = (struct ubsec_pktctx_3des *)
1782	((u_int8_t *)(dmap->d_alloc.dma_vaddr) +
1783	offsetof(struct ubsec_dmachunk, d_ctx));
1784
1785	ph = (struct ubsec_pktctx_hdr *)ctx;
1786	ph->ph_len = htole16(sizeof(*ctx));
1787	ph->ph_type = htole16(UBS_PKTCTX_TYPE_IPSEC_3DES);
1788	ph->ph_flags = flags;
1789	ph->ph_offset = htole16(coffset >> `2`);
1790
1791	for (i = `0`; i < `6`; i++)
1792	ctx->pc_deskey[i] = key.ses_key[i];
1793	for (i = `0`; i < `5`; i++)
1794	ctx->pc_hminner[i] = key.ses_hminner[i];
1795	for (i = `0`; i < `5`; i++)
1796	ctx->pc_hmouter[i] = key.ses_hmouter[i];
1797	for (i = `0`; i < `2`; i++)
1798	ctx->pc_iv[i] = key.ses_iv[i];
1799	} else {
1800	struct ubsec_pktctx ctx = (struct* ubsec_pktctx *)
1801	((u_int8_t *)dmap->d_alloc.dma_vaddr +
1802	offsetof(struct ubsec_dmachunk, d_ctx));
1803
1804	ctx->pc_flags = flags;
1805	ctx->pc_offset = htole16(coffset >> `2`);
1806	for (i = `0`; i < `6`; i++)
1807	ctx->pc_deskey[i] = key.ses_key[i];
1808	for (i = `0`; i < `5`; i++)
1809	ctx->pc_hminner[i] = key.ses_hminner[i];
1810	for (i = `0`; i < `5`; i++)
1811	ctx->pc_hmouter[i] = key.ses_hmouter[i];
1812	for (i = `0`; i < `2`; i++)
1813	ctx->pc_iv[i] = key.ses_iv[i];
1814	}
1815
1816	mutex_spin_enter(&sc->sc_mtx);
1817	SIMPLEQ_INSERT_TAIL(&sc->sc_queue, q, q_next);
1818	sc->sc_nqueue++;
1819	ubsecstats.hst_ipackets++;
1820	ubsecstats.hst_ibytes += dmap->d_alloc.dma_map->dm_mapsize;
1821	if ((hint & CRYPTO_HINT_MORE) == `0` \|\| sc->sc_nqueue >= ubsec_maxbatch)
1822	ubsec_feed(sc);
1823	mutex_spin_exit(&sc->sc_mtx);
1824	return (`0`);
1825
1826	errout:
1827	if (q != NULL) {
1828	if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m))
1829	m_freem(q->q_dst_m);
1830
1831	if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) {
1832	bus_dmamap_unload(sc->sc_dmat, q->q_dst_map);
1833	}
1834	if (q->q_src_map != NULL) {
1835	bus_dmamap_unload(sc->sc_dmat, q->q_src_map);
1836	}
1837
1838	mutex_spin_enter(&sc->sc_mtx);
1839	SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
1840	mutex_spin_exit(&sc->sc_mtx);
1841	}
1842	#if 0 /* jonathan says: this openbsd code seems to be subsumed elsewhere */
1843	if (err == EINVAL)
1844	ubsecstats.hst_invalid++;
1845	else
1846	ubsecstats.hst_nomem++;
1847	#endif
1848	if (err != ERESTART) {
1849	crp->crp_etype = err;
1850	crypto_done(crp);
1851	} else {
1852	sc->sc_needwakeup \|= CRYPTO_SYMQ;
1853	}
1854	return (err);
1855	}
1856
1857	static void
1858	ubsec_callback(struct ubsec_softc sc, struct* ubsec_q *q)
1859	{
1860	struct cryptop crp = (struct* cryptop *)q->q_crp;
1861	struct cryptodesc *crd;
1862	struct ubsec_dma *dmap = q->q_dma;
1863
1864	ubsecstats.hst_opackets++;
1865	ubsecstats.hst_obytes += dmap->d_alloc.dma_size;
1866
1867	bus_dmamap_sync(sc->sc_dmat, dmap->d_alloc.dma_map, `0`,
1868	dmap->d_alloc.dma_map->dm_mapsize,
1869	BUS_DMASYNC_POSTREAD\|BUS_DMASYNC_POSTWRITE);
1870	if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) {
1871	bus_dmamap_sync(sc->sc_dmat, q->q_dst_map,
1872	`0`, q->q_dst_map->dm_mapsize, BUS_DMASYNC_POSTREAD);
1873	bus_dmamap_unload(sc->sc_dmat, q->q_dst_map);
1874	}
1875	bus_dmamap_sync(sc->sc_dmat, q->q_src_map,
1876	`0`, q->q_src_map->dm_mapsize, BUS_DMASYNC_POSTWRITE);
1877	bus_dmamap_unload(sc->sc_dmat, q->q_src_map);
1878
1879	if ((crp->crp_flags & CRYPTO_F_IMBUF) && (q->q_src_m != q->q_dst_m)) {
1880	m_freem(q->q_src_m);
1881	crp->crp_buf = (void *)q->q_dst_m;
1882	}
1883
1884	/ copy out IV for future use /
1885	if (q->q_flags & UBSEC_QFLAGS_COPYOUTIV) {
1886	for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
1887	if (crd->crd_alg != CRYPTO_DES_CBC &&
1888	crd->crd_alg != CRYPTO_3DES_CBC &&
1889	crd->crd_alg != CRYPTO_AES_CBC)
1890	continue;
1891	if (crp->crp_flags & CRYPTO_F_IMBUF)
1892	m_copydata((struct mbuf *)crp->crp_buf,
1893	crd->crd_skip + crd->crd_len - `8`, `8`,
1894	(void *)sc->sc_sessions[q->q_sesn].ses_iv);
1895	else if (crp->crp_flags & CRYPTO_F_IOV) {
1896	cuio_copydata((struct uio *)crp->crp_buf,
1897	crd->crd_skip + crd->crd_len - `8`, `8`,
1898	(void *)sc->sc_sessions[q->q_sesn].ses_iv);
1899	}
1900	break;
1901	}
1902	}
1903
1904	for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
1905	if (crd->crd_alg != CRYPTO_MD5_HMAC_96 &&
1906	crd->crd_alg != CRYPTO_SHA1_HMAC_96)
1907	continue;
1908	if (crp->crp_flags & CRYPTO_F_IMBUF)
1909	m_copyback((struct mbuf *)crp->crp_buf,
1910	crd->crd_inject, `12`,
1911	(void *)dmap->d_dma->d_macbuf);
1912	else if (crp->crp_flags & CRYPTO_F_IOV && crp->crp_mac)
1913	bcopy((void *)dmap->d_dma->d_macbuf,
1914	crp->crp_mac, `12`);
1915	break;
1916	}
1917	SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
1918	crypto_done(crp);
1919	}
1920
1921	static void
1922	ubsec_mcopy(struct mbuf srcm, struct* mbuf dstm, int* hoffset, int toffset)
1923	{
1924	int i, j, dlen, slen;
1925	char dptr, sptr;
1926
1927	j = `0`;
1928	sptr = srcm->m_data;
1929	slen = srcm->m_len;
1930	dptr = dstm->m_data;
1931	dlen = dstm->m_len;
1932
1933	while (`1`) {
1934	for (i = `0`; i < min(slen, dlen); i++) {
1935	if (j < hoffset \|\| j >= toffset)
1936	dptr++ = sptr++;
1937	slen--;
1938	dlen--;
1939	j++;
1940	}
1941	if (slen == `0`) {
1942	srcm = srcm->m_next;
1943	if (srcm == NULL)
1944	return;
1945	sptr = srcm->m_data;
1946	slen = srcm->m_len;
1947	}
1948	if (dlen == `0`) {
1949	dstm = dstm->m_next;
1950	if (dstm == NULL)
1951	return;
1952	dptr = dstm->m_data;
1953	dlen = dstm->m_len;
1954	}
1955	}
1956	}
1957
1958	/*
1959	* feed the key generator, must be called at splnet() or higher.
1960	*/
1961	static void
1962	ubsec_feed2(struct ubsec_softc *sc)
1963	{
1964	struct ubsec_q2 *q;
1965
1966	while (!SIMPLEQ_EMPTY(&sc->sc_queue2)) {
1967	if (READ_REG(sc, BS_STAT) & BS_STAT_MCR2_FULL)
1968	break;
1969	q = SIMPLEQ_FIRST(&sc->sc_queue2);
1970
1971	bus_dmamap_sync(sc->sc_dmat, q->q_mcr.dma_map, `0`,
1972	q->q_mcr.dma_map->dm_mapsize,
1973	BUS_DMASYNC_PREREAD \| BUS_DMASYNC_PREWRITE);
1974	bus_dmamap_sync(sc->sc_dmat, q->q_ctx.dma_map, `0`,
1975	q->q_ctx.dma_map->dm_mapsize,
1976	BUS_DMASYNC_PREWRITE);
1977
1978	WRITE_REG(sc, BS_MCR2, q->q_mcr.dma_paddr);
1979	q = SIMPLEQ_FIRST(&sc->sc_queue2);
1980	SIMPLEQ_REMOVE_HEAD(&sc->sc_queue2, /q,/ q_next);
1981	--sc->sc_nqueue2;
1982	SIMPLEQ_INSERT_TAIL(&sc->sc_qchip2, q, q_next);
1983	}
1984	}
1985
1986	/*
1987	* feed the RNG (used instead of ubsec_feed2() on 5827+ devices)
1988	*/
1989	void
1990	ubsec_feed4(struct ubsec_softc *sc)
1991	{
1992	struct ubsec_q2 *q;
1993
1994	while (!SIMPLEQ_EMPTY(&sc->sc_queue4)) {
1995	if (READ_REG(sc, BS_STAT) & BS_STAT_MCR4_FULL)
1996	break;
1997	q = SIMPLEQ_FIRST(&sc->sc_queue4);
1998
1999	bus_dmamap_sync(sc->sc_dmat, q->q_mcr.dma_map, `0`,
2000	q->q_mcr.dma_map->dm_mapsize,
2001	BUS_DMASYNC_PREREAD \| BUS_DMASYNC_PREWRITE);
2002	bus_dmamap_sync(sc->sc_dmat, q->q_ctx.dma_map, `0`,
2003	q->q_ctx.dma_map->dm_mapsize,
2004	BUS_DMASYNC_PREWRITE);
2005
2006	WRITE_REG(sc, BS_MCR4, q->q_mcr.dma_paddr);
2007	SIMPLEQ_REMOVE_HEAD(&sc->sc_queue4, q_next);
2008	--sc->sc_nqueue4;
2009	SIMPLEQ_INSERT_TAIL(&sc->sc_qchip4, q, q_next);
2010	}
2011	}
2012
2013	/*
2014	* Callback for handling random numbers
2015	*/
2016	static void
2017	ubsec_callback2(struct ubsec_softc sc, struct* ubsec_q2 *q)
2018	{
2019	struct cryptkop *krp;
2020	struct ubsec_ctx_keyop *ctx;
2021
2022	ctx = (struct ubsec_ctx_keyop *)q->q_ctx.dma_vaddr;
2023	bus_dmamap_sync(sc->sc_dmat, q->q_ctx.dma_map, `0`,
2024	q->q_ctx.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE);
2025
2026	switch (q->q_type) {
2027	#ifndef UBSEC_NO_RNG
2028	case UBS_CTXOP_RNGSHA1:
2029	case UBS_CTXOP_RNGBYPASS: {
2030	struct ubsec_q2_rng rng = (struct* ubsec_q2_rng *)q;
2031	u_int32_t *p;
2032	int i;
2033
2034	bus_dmamap_sync(sc->sc_dmat, rng->rng_buf.dma_map, `0`,
2035	rng->rng_buf.dma_map->dm_mapsize, BUS_DMASYNC_POSTREAD);
2036	p = (u_int32_t *)rng->rng_buf.dma_vaddr;
2037	#ifndef __NetBSD__
2038	for (i = `0`; i < UBSEC_RNG_BUFSIZ; p++, i++)
2039	add_true_randomness(letoh32(*p));
2040	#else
2041	i = UBSEC_RNG_BUFSIZ * sizeof(u_int32_t);
2042	rnd_add_data(&sc->sc_rnd_source, (char )p, i, i NBBY);
2043	sc->sc_rng_need -= i;
2044	#endif
2045	rng->rng_used = `0`;
2046	#ifdef __OpenBSD__
2047	timeout_add(&sc->sc_rngto, sc->sc_rnghz);
2048	#else
2049	if (sc->sc_rng_need > `0`) {
2050	callout_schedule(&sc->sc_rngto, sc->sc_rnghz);
2051	}
2052	#endif
2053	break;
2054	}
2055	#endif
2056	case UBS_CTXOP_MODEXP: {
2057	struct ubsec_q2_modexp me = (struct* ubsec_q2_modexp *)q;
2058	u_int rlen, clen;
2059
2060	krp = me->me_krp;
2061	rlen = (me->me_modbits + `7`) / `8`;
2062	clen = (krp->krp_param[krp->krp_iparams].crp_nbits + `7`) / `8`;
2063
2064	bus_dmamap_sync(sc->sc_dmat, me->me_M.dma_map,
2065	`0`, me->me_M.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE);
2066	bus_dmamap_sync(sc->sc_dmat, me->me_E.dma_map,
2067	`0`, me->me_E.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE);
2068	bus_dmamap_sync(sc->sc_dmat, me->me_C.dma_map,
2069	`0`, me->me_C.dma_map->dm_mapsize, BUS_DMASYNC_POSTREAD);
2070	bus_dmamap_sync(sc->sc_dmat, me->me_epb.dma_map,
2071	`0`, me->me_epb.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE);
2072
2073	if (clen < rlen)
2074	krp->krp_status = E2BIG;
2075	else {
2076	if (sc->sc_flags & UBS_FLAGS_HWNORM) {
2077	memset(krp->krp_param[krp->krp_iparams].crp_p, `0`,
2078	(krp->krp_param[krp->krp_iparams].crp_nbits
2079	+ `7`) / `8`);
2080	bcopy(me->me_C.dma_vaddr,
2081	krp->krp_param[krp->krp_iparams].crp_p,
2082	(me->me_modbits + `7`) / `8`);
2083	} else
2084	ubsec_kshift_l(me->me_shiftbits,
2085	me->me_C.dma_vaddr, me->me_normbits,
2086	krp->krp_param[krp->krp_iparams].crp_p,
2087	krp->krp_param[krp->krp_iparams].crp_nbits);
2088	}
2089
2090	crypto_kdone(krp);
2091
2092	/ bzero all potentially sensitive data /
2093	memset(me->me_E.dma_vaddr, `0`, me->me_E.dma_size);
2094	memset(me->me_M.dma_vaddr, `0`, me->me_M.dma_size);
2095	memset(me->me_C.dma_vaddr, `0`, me->me_C.dma_size);
2096	memset(me->me_q.q_ctx.dma_vaddr, `0`, me->me_q.q_ctx.dma_size);
2097
2098	/ Can't free here, so put us on the free list. /
2099	SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &me->me_q, q_next);
2100	break;
2101	}
2102	case UBS_CTXOP_RSAPRIV: {
2103	struct ubsec_q2_rsapriv rp = (struct* ubsec_q2_rsapriv *)q;
2104	u_int len;
2105
2106	krp = rp->rpr_krp;
2107	bus_dmamap_sync(sc->sc_dmat, rp->rpr_msgin.dma_map, `0`,
2108	rp->rpr_msgin.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE);
2109	bus_dmamap_sync(sc->sc_dmat, rp->rpr_msgout.dma_map, `0`,
2110	rp->rpr_msgout.dma_map->dm_mapsize, BUS_DMASYNC_POSTREAD);
2111
2112	len = (krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_nbits + `7`)
2113	/ `8`;
2114	bcopy(rp->rpr_msgout.dma_vaddr,
2115	krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_p, len);
2116
2117	crypto_kdone(krp);
2118
2119	memset(rp->rpr_msgin.dma_vaddr, `0`, rp->rpr_msgin.dma_size);
2120	memset(rp->rpr_msgout.dma_vaddr, `0`, rp->rpr_msgout.dma_size);
2121	memset(rp->rpr_q.q_ctx.dma_vaddr, `0`, rp->rpr_q.q_ctx.dma_size);
2122
2123	/ Can't free here, so put us on the free list. /
2124	SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &rp->rpr_q, q_next);
2125	break;
2126	}
2127	default:
2128	printf("%s: unknown ctx op: %x\n", device_xname(sc->sc_dev),
2129	letoh16(ctx->ctx_op));
2130	break;
2131	}
2132	}
2133
2134	#ifndef UBSEC_NO_RNG
2135
2136	static void
2137	ubsec_rng_get(size_t bytes, void *vsc)
2138	{
2139	struct ubsec_softc *sc = vsc;
2140
2141	mutex_spin_enter(&sc->sc_mtx);
2142	sc->sc_rng_need = bytes;
2143	ubsec_rng_locked(sc);
2144	mutex_spin_exit(&sc->sc_mtx);
2145
2146	}
2147
2148	static void
2149	ubsec_rng(void *vsc)
2150	{
2151	struct ubsec_softc *sc = vsc;
2152	mutex_spin_enter(&sc->sc_mtx);
2153	ubsec_rng_locked(sc);
2154	mutex_spin_exit(&sc->sc_mtx);
2155	}
2156
2157	static void
2158	ubsec_rng_locked(void *vsc)
2159	{
2160	struct ubsec_softc *sc = vsc;
2161	struct ubsec_q2_rng *rng = &sc->sc_rng;
2162	struct ubsec_mcr *mcr;
2163	struct ubsec_ctx_rngbypass *ctx;
2164	int *nqueue;
2165
2166	/ Caller is responsible to lock and release sc_mtx. /
2167	KASSERT(mutex_owned(&sc->sc_mtx));
2168
2169	if (rng->rng_used) {
2170	return;
2171	}
2172
2173	if (sc->sc_rng_need < `1`) {
2174	callout_stop(&sc->sc_rngto);
2175	return;
2176	}
2177
2178	if (sc->sc_flags & UBS_FLAGS_RNG4)
2179	nqueue = &sc->sc_nqueue4;
2180	else
2181	nqueue = &sc->sc_nqueue2;
2182
2183	(*nqueue)++;
2184	if (*nqueue >= UBS_MAX_NQUEUE)
2185	goto out;
2186
2187	mcr = (struct ubsec_mcr *)rng->rng_q.q_mcr.dma_vaddr;
2188	ctx = (struct ubsec_ctx_rngbypass *)rng->rng_q.q_ctx.dma_vaddr;
2189
2190	mcr->mcr_pkts = htole16(`1`);
2191	mcr->mcr_flags = `0`;
2192	mcr->mcr_cmdctxp = htole32(rng->rng_q.q_ctx.dma_paddr);
2193	mcr->mcr_ipktbuf.pb_addr = mcr->mcr_ipktbuf.pb_next = `0`;
2194	mcr->mcr_ipktbuf.pb_len = `0`;
2195	mcr->mcr_reserved = mcr->mcr_pktlen = `0`;
2196	mcr->mcr_opktbuf.pb_addr = htole32(rng->rng_buf.dma_paddr);
2197	mcr->mcr_opktbuf.pb_len = htole32(((sizeof(u_int32_t) * UBSEC_RNG_BUFSIZ)) &
2198	UBS_PKTBUF_LEN);
2199	mcr->mcr_opktbuf.pb_next = `0`;
2200
2201	ctx->rbp_len = htole16(sizeof(struct ubsec_ctx_rngbypass));
2202	ctx->rbp_op = htole16(UBS_CTXOP_RNGSHA1);
2203	rng->rng_q.q_type = UBS_CTXOP_RNGSHA1;
2204
2205	bus_dmamap_sync(sc->sc_dmat, rng->rng_buf.dma_map, `0`,
2206	rng->rng_buf.dma_map->dm_mapsize, BUS_DMASYNC_PREREAD);
2207
2208	if (sc->sc_flags & UBS_FLAGS_RNG4) {
2209	SIMPLEQ_INSERT_TAIL(&sc->sc_queue4, &rng->rng_q, q_next);
2210	ubsec_feed4(sc);
2211	} else {
2212	SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rng->rng_q, q_next);
2213	ubsec_feed2(sc);
2214	}
2215	rng->rng_used = `1`;
2216	ubsecstats.hst_rng++;
2217
2218	return;
2219
2220	out:
2221	/*
2222	* Something weird happened, generate our own call back.
2223	*/
2224	(*nqueue)--;
2225	#ifdef __OpenBSD__
2226	timeout_add(&sc->sc_rngto, sc->sc_rnghz);
2227	#else
2228	callout_schedule(&sc->sc_rngto, sc->sc_rnghz);
2229	#endif
2230	}
2231	#endif /* UBSEC_NO_RNG */
2232
2233	static int
2234	ubsec_dma_malloc(struct ubsec_softc *sc, bus_size_t size,
2235	struct ubsec_dma_alloc dma,int* mapflags)
2236	{
2237	int r;
2238
2239	if ((r = bus_dmamem_alloc(sc->sc_dmat, size, PAGE_SIZE, `0`,
2240	&dma->dma_seg, `1`, &dma->dma_nseg, BUS_DMA_NOWAIT)) != `0`)
2241	goto fail_0;
2242
2243	if ((r = bus_dmamem_map(sc->sc_dmat, &dma->dma_seg, dma->dma_nseg,
2244	size, &dma->dma_vaddr, mapflags \| BUS_DMA_NOWAIT)) != `0`)
2245	goto fail_1;
2246
2247	if ((r = bus_dmamap_create(sc->sc_dmat, size, `1`, size, `0`,
2248	BUS_DMA_NOWAIT, &dma->dma_map)) != `0`)
2249	goto fail_2;
2250
2251	if ((r = bus_dmamap_load(sc->sc_dmat, dma->dma_map, dma->dma_vaddr,
2252	size, NULL, BUS_DMA_NOWAIT)) != `0`)
2253	goto fail_3;
2254
2255	dma->dma_paddr = dma->dma_map->dm_segs[`0`].ds_addr;
2256	dma->dma_size = size;
2257	return (`0`);
2258
2259	fail_3:
2260	bus_dmamap_destroy(sc->sc_dmat, dma->dma_map);
2261	fail_2:
2262	bus_dmamem_unmap(sc->sc_dmat, dma->dma_vaddr, size);
2263	fail_1:
2264	bus_dmamem_free(sc->sc_dmat, &dma->dma_seg, dma->dma_nseg);
2265	fail_0:
2266	dma->dma_map = NULL;
2267	return (r);
2268	}
2269
2270	static void
2271	ubsec_dma_free(struct ubsec_softc sc, struct* ubsec_dma_alloc *dma)
2272	{
2273	bus_dmamap_unload(sc->sc_dmat, dma->dma_map);
2274	bus_dmamem_unmap(sc->sc_dmat, dma->dma_vaddr, dma->dma_size);
2275	bus_dmamem_free(sc->sc_dmat, &dma->dma_seg, dma->dma_nseg);
2276	bus_dmamap_destroy(sc->sc_dmat, dma->dma_map);
2277	}
2278
2279	/*
2280	* Resets the board. Values in the regesters are left as is
2281	* from the reset (i.e. initial values are assigned elsewhere).
2282	*/
2283	static void
2284	ubsec_reset_board(struct ubsec_softc *sc)
2285	{
2286	volatile u_int32_t ctrl;
2287
2288	ctrl = READ_REG(sc, BS_CTRL);
2289	ctrl \|= BS_CTRL_RESET;
2290	WRITE_REG(sc, BS_CTRL, ctrl);
2291
2292	/*
2293	* Wait aprox. 30 PCI clocks = 900 ns = 0.9 us
2294	*/
2295	DELAY(`10`);
2296
2297	/ Enable RNG and interrupts on newer devices /
2298	if (sc->sc_flags & UBS_FLAGS_MULTIMCR) {
2299	#ifndef UBSEC_NO_RNG
2300	WRITE_REG(sc, BS_CFG, BS_CFG_RNG);
2301	#endif
2302	WRITE_REG(sc, BS_INT, BS_INT_DMAINT);
2303	}
2304	}
2305
2306	/*
2307	* Init Broadcom registers
2308	*/
2309	static void
2310	ubsec_init_board(struct ubsec_softc *sc)
2311	{
2312	u_int32_t ctrl;
2313
2314	ctrl = READ_REG(sc, BS_CTRL);
2315	ctrl &= ~(BS_CTRL_BE32 \| BS_CTRL_BE64);
2316	ctrl \|= BS_CTRL_LITTLE_ENDIAN \| BS_CTRL_MCR1INT;
2317
2318	/*
2319	* XXX: Sam Leffler's code has (UBS_FLAGS_KEY\|UBS_FLAGS_RNG)).
2320	* anyone got hw docs?
2321	*/
2322	if (sc->sc_flags & UBS_FLAGS_KEY)
2323	ctrl \|= BS_CTRL_MCR2INT;
2324	else
2325	ctrl &= ~BS_CTRL_MCR2INT;
2326
2327	if (sc->sc_flags & UBS_FLAGS_HWNORM)
2328	ctrl &= ~BS_CTRL_SWNORM;
2329
2330	if (sc->sc_flags & UBS_FLAGS_MULTIMCR) {
2331	ctrl \|= BS_CTRL_BSIZE240;
2332	ctrl &= ~BS_CTRL_MCR3INT; / MCR3 is reserved for SSL /
2333
2334	if (sc->sc_flags & UBS_FLAGS_RNG4)
2335	ctrl \|= BS_CTRL_MCR4INT;
2336	else
2337	ctrl &= ~BS_CTRL_MCR4INT;
2338	}
2339
2340	WRITE_REG(sc, BS_CTRL, ctrl);
2341	}
2342
2343	/*
2344	* Init Broadcom PCI registers
2345	*/
2346	static void
2347	ubsec_init_pciregs(struct pci_attach_args *pa)
2348	{
2349	pci_chipset_tag_t pc = pa->pa_pc;
2350	u_int32_t misc;
2351
2352	/*
2353	* This will set the cache line size to 1, this will
2354	* force the BCM58xx chip just to do burst read/writes.
2355	* Cache line read/writes are to slow
2356	*/
2357	misc = pci_conf_read(pc, pa->pa_tag, PCI_BHLC_REG);
2358	misc = (misc & ~(PCI_CACHELINE_MASK << PCI_CACHELINE_SHIFT))
2359	\| ((UBS_DEF_CACHELINE & `0xff`) << PCI_CACHELINE_SHIFT);
2360	pci_conf_write(pc, pa->pa_tag, PCI_BHLC_REG, misc);
2361	}
2362
2363	/*
2364	* Clean up after a chip crash.
2365	* It is assumed that the caller in splnet()
2366	*/
2367	static void
2368	ubsec_cleanchip(struct ubsec_softc *sc)
2369	{
2370	struct ubsec_q *q;
2371
2372	while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) {
2373	q = SIMPLEQ_FIRST(&sc->sc_qchip);
2374	SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, /q,/ q_next);
2375	ubsec_free_q(sc, q);
2376	}
2377	sc->sc_nqchip = `0`;
2378	}
2379
2380	/*
2381	* free a ubsec_q
2382	* It is assumed that the caller is within splnet()
2383	*/
2384	static int
2385	ubsec_free_q(struct ubsec_softc sc, struct* ubsec_q *q)
2386	{
2387	struct ubsec_q *q2;
2388	struct cryptop *crp;
2389	int npkts;
2390	int i;
2391
2392	npkts = q->q_nstacked_mcrs;
2393
2394	for (i = `0`; i < npkts; i++) {
2395	if(q->q_stacked_mcr[i]) {
2396	q2 = q->q_stacked_mcr[i];
2397
2398	if ((q2->q_dst_m != NULL)
2399	&& (q2->q_src_m != q2->q_dst_m))
2400	m_freem(q2->q_dst_m);
2401
2402	crp = (struct cryptop *)q2->q_crp;
2403
2404	SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q2, q_next);
2405
2406	crp->crp_etype = EFAULT;
2407	crypto_done(crp);
2408	} else {
2409	break;
2410	}
2411	}
2412
2413	/*
2414	* Free header MCR
2415	*/
2416	if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m))
2417	m_freem(q->q_dst_m);
2418
2419	crp = (struct cryptop *)q->q_crp;
2420
2421	SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
2422
2423	crp->crp_etype = EFAULT;
2424	crypto_done(crp);
2425	return(`0`);
2426	}
2427
2428	/*
2429	* Routine to reset the chip and clean up.
2430	* It is assumed that the caller is in splnet()
2431	*/
2432	static void
2433	ubsec_totalreset(struct ubsec_softc *sc)
2434	{
2435	ubsec_reset_board(sc);
2436	ubsec_init_board(sc);
2437	ubsec_cleanchip(sc);
2438	}
2439
2440	static int
2441	ubsec_dmamap_aligned(bus_dmamap_t map)
2442	{
2443	int i;
2444
2445	for (i = `0`; i < map->dm_nsegs; i++) {
2446	if (map->dm_segs[i].ds_addr & `3`)
2447	return (`0`);
2448	if ((i != (map->dm_nsegs - `1`)) &&
2449	(map->dm_segs[i].ds_len & `3`))
2450	return (`0`);
2451	}
2452	return (`1`);
2453	}
2454
2455	#ifdef __OpenBSD__
2456	struct ubsec_softc *
2457	ubsec_kfind(struct cryptkop *krp)
2458	{
2459	struct ubsec_softc *sc;
2460	int i;
2461
2462	for (i = `0`; i < ubsec_cd.cd_ndevs; i++) {
2463	sc = ubsec_cd.cd_devs[i];
2464	if (sc == NULL)
2465	continue;
2466	if (sc->sc_cid == krp->krp_hid)
2467	return (sc);
2468	}
2469	return (NULL);
2470	}
2471	#endif
2472
2473	static void
2474	ubsec_kfree(struct ubsec_softc sc, struct* ubsec_q2 *q)
2475	{
2476	switch (q->q_type) {
2477	case UBS_CTXOP_MODEXP: {
2478	struct ubsec_q2_modexp me = (struct* ubsec_q2_modexp *)q;
2479
2480	ubsec_dma_free(sc, &me->me_q.q_mcr);
2481	ubsec_dma_free(sc, &me->me_q.q_ctx);
2482	ubsec_dma_free(sc, &me->me_M);
2483	ubsec_dma_free(sc, &me->me_E);
2484	ubsec_dma_free(sc, &me->me_C);
2485	ubsec_dma_free(sc, &me->me_epb);
2486	free(me, M_DEVBUF);
2487	break;
2488	}
2489	case UBS_CTXOP_RSAPRIV: {
2490	struct ubsec_q2_rsapriv rp = (struct* ubsec_q2_rsapriv *)q;
2491
2492	ubsec_dma_free(sc, &rp->rpr_q.q_mcr);
2493	ubsec_dma_free(sc, &rp->rpr_q.q_ctx);
2494	ubsec_dma_free(sc, &rp->rpr_msgin);
2495	ubsec_dma_free(sc, &rp->rpr_msgout);
2496	free(rp, M_DEVBUF);
2497	break;
2498	}
2499	default:
2500	printf("%s: invalid kfree 0x%x\n", device_xname(sc->sc_dev),
2501	q->q_type);
2502	break;
2503	}
2504	}
2505
2506	static int
2507	ubsec_kprocess(void arg, struct* cryptkop krp, int* hint)
2508	{
2509	struct ubsec_softc *sc;
2510	int r;
2511
2512	if (krp == NULL \|\| krp->krp_callback == NULL)
2513	return (EINVAL);
2514	#ifdef __OpenBSD__
2515	if ((sc = ubsec_kfind(krp)) == NULL)
2516	return (EINVAL);
2517	#else
2518	sc = arg;
2519	KASSERT(sc != NULL /, ("ubsec_kprocess: null softc")/);
2520	#endif
2521
2522	while (!SIMPLEQ_EMPTY(&sc->sc_q2free)) {
2523	struct ubsec_q2 *q;
2524
2525	q = SIMPLEQ_FIRST(&sc->sc_q2free);
2526	SIMPLEQ_REMOVE_HEAD(&sc->sc_q2free, /q,/ q_next);
2527	ubsec_kfree(sc, q);
2528	}
2529
2530	switch (krp->krp_op) {
2531	case CRK_MOD_EXP:
2532	if (sc->sc_flags & UBS_FLAGS_HWNORM)
2533	r = ubsec_kprocess_modexp_hw(sc, krp, hint);
2534	else
2535	r = ubsec_kprocess_modexp_sw(sc, krp, hint);
2536	break;
2537	case CRK_MOD_EXP_CRT:
2538	r = ubsec_kprocess_rsapriv(sc, krp, hint);
2539	break;
2540	default:
2541	printf("%s: kprocess: invalid op 0x%x\n",
2542	device_xname(sc->sc_dev), krp->krp_op);
2543	krp->krp_status = EOPNOTSUPP;
2544	crypto_kdone(krp);
2545	r = `0`;
2546	}
2547	return (r);
2548	}
2549
2550	/*
2551	* Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (sw normalization)
2552	*/
2553	static int
2554	ubsec_kprocess_modexp_sw(struct ubsec_softc sc, struct* cryptkop *krp,
2555	int hint)
2556	{
2557	struct ubsec_q2_modexp *me;
2558	struct ubsec_mcr *mcr;
2559	struct ubsec_ctx_modexp *ctx;
2560	struct ubsec_pktbuf *epb;
2561	int err = `0`;
2562	u_int nbits, normbits, mbits, shiftbits, ebits;
2563
2564	me = (struct ubsec_q2_modexp )malloc(sizeof* *me, M_DEVBUF, M_NOWAIT);
2565	if (me == NULL) {
2566	err = ENOMEM;
2567	goto errout;
2568	}
2569	memset(me, `0`, sizeof *me);
2570	me->me_krp = krp;
2571	me->me_q.q_type = UBS_CTXOP_MODEXP;
2572
2573	nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]);
2574	if (nbits <= `512`)
2575	normbits = `512`;
2576	else if (nbits <= `768`)
2577	normbits = `768`;
2578	else if (nbits <= `1024`)
2579	normbits = `1024`;
2580	else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= `1536`)
2581	normbits = `1536`;
2582	else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= `2048`)
2583	normbits = `2048`;
2584	else {
2585	err = E2BIG;
2586	goto errout;
2587	}
2588
2589	shiftbits = normbits - nbits;
2590
2591	me->me_modbits = nbits;
2592	me->me_shiftbits = shiftbits;
2593	me->me_normbits = normbits;
2594
2595	/ Sanity check: result bits must be >= true modulus bits. /
2596	if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) {
2597	err = ERANGE;
2598	goto errout;
2599	}
2600
2601	if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
2602	&me->me_q.q_mcr, `0`)) {
2603	err = ENOMEM;
2604	goto errout;
2605	}
2606	mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr;
2607
2608	if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp),
2609	&me->me_q.q_ctx, `0`)) {
2610	err = ENOMEM;
2611	goto errout;
2612	}
2613
2614	mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]);
2615	if (mbits > nbits) {
2616	err = E2BIG;
2617	goto errout;
2618	}
2619	if (ubsec_dma_malloc(sc, normbits / `8`, &me->me_M, `0`)) {
2620	err = ENOMEM;
2621	goto errout;
2622	}
2623	ubsec_kshift_r(shiftbits,
2624	krp->krp_param[UBS_MODEXP_PAR_M].crp_p, mbits,
2625	me->me_M.dma_vaddr, normbits);
2626
2627	if (ubsec_dma_malloc(sc, normbits / `8`, &me->me_C, `0`)) {
2628	err = ENOMEM;
2629	goto errout;
2630	}
2631	memset(me->me_C.dma_vaddr, `0`, me->me_C.dma_size);
2632
2633	ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]);
2634	if (ebits > nbits) {
2635	err = E2BIG;
2636	goto errout;
2637	}
2638	if (ubsec_dma_malloc(sc, normbits / `8`, &me->me_E, `0`)) {
2639	err = ENOMEM;
2640	goto errout;
2641	}
2642	ubsec_kshift_r(shiftbits,
2643	krp->krp_param[UBS_MODEXP_PAR_E].crp_p, ebits,
2644	me->me_E.dma_vaddr, normbits);
2645
2646	if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf),
2647	&me->me_epb, `0`)) {
2648	err = ENOMEM;
2649	goto errout;
2650	}
2651	epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr;
2652	epb->pb_addr = htole32(me->me_E.dma_paddr);
2653	epb->pb_next = `0`;
2654	epb->pb_len = htole32(normbits / `8`);
2655
2656	#ifdef UBSEC_DEBUG
2657	if (ubsec_debug) {
2658	printf("Epb ");
2659	ubsec_dump_pb(epb);
2660	}
2661	#endif
2662
2663	mcr->mcr_pkts = htole16(`1`);
2664	mcr->mcr_flags = `0`;
2665	mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr);
2666	mcr->mcr_reserved = `0`;
2667	mcr->mcr_pktlen = `0`;
2668
2669	mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr);
2670	mcr->mcr_ipktbuf.pb_len = htole32(normbits / `8`);
2671	mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr);
2672
2673	mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr);
2674	mcr->mcr_opktbuf.pb_next = `0`;
2675	mcr->mcr_opktbuf.pb_len = htole32(normbits / `8`);
2676
2677	#ifdef DIAGNOSTIC
2678	/ Misaligned output buffer will hang the chip. /
2679	if ((letoh32(mcr->mcr_opktbuf.pb_addr) & `3`) != `0`)
2680	panic("%s: modexp invalid addr 0x%x", device_xname(sc->sc_dev),
2681	letoh32(mcr->mcr_opktbuf.pb_addr));
2682	if ((letoh32(mcr->mcr_opktbuf.pb_len) & `3`) != `0`)
2683	panic("%s: modexp invalid len 0x%x", device_xname(sc->sc_dev),
2684	letoh32(mcr->mcr_opktbuf.pb_len));
2685	#endif
2686
2687	ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr;
2688	memset(ctx, `0`, sizeof(*ctx));
2689	ubsec_kshift_r(shiftbits,
2690	krp->krp_param[UBS_MODEXP_PAR_N].crp_p, nbits,
2691	ctx->me_N, normbits);
2692	ctx->me_len = htole16((normbits / `8`) + (`4` * sizeof(u_int16_t)));
2693	ctx->me_op = htole16(UBS_CTXOP_MODEXP);
2694	ctx->me_E_len = htole16(nbits);
2695	ctx->me_N_len = htole16(nbits);
2696
2697	#ifdef UBSEC_DEBUG
2698	if (ubsec_debug) {
2699	ubsec_dump_mcr(mcr);
2700	ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx);
2701	}
2702	#endif
2703
2704	/*
2705	* ubsec_feed2 will sync mcr and ctx, we just need to sync
2706	* everything else.
2707	*/
2708	bus_dmamap_sync(sc->sc_dmat, me->me_M.dma_map,
2709	`0`, me->me_M.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
2710	bus_dmamap_sync(sc->sc_dmat, me->me_E.dma_map,
2711	`0`, me->me_E.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
2712	bus_dmamap_sync(sc->sc_dmat, me->me_C.dma_map,
2713	`0`, me->me_C.dma_map->dm_mapsize, BUS_DMASYNC_PREREAD);
2714	bus_dmamap_sync(sc->sc_dmat, me->me_epb.dma_map,
2715	`0`, me->me_epb.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
2716
2717	/ Enqueue and we're done... /
2718	mutex_spin_enter(&sc->sc_mtx);
2719	SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next);
2720	ubsec_feed2(sc);
2721	ubsecstats.hst_modexp++;
2722	mutex_spin_exit(&sc->sc_mtx);
2723
2724	return (`0`);
2725
2726	errout:
2727	if (me != NULL) {
2728	if (me->me_q.q_mcr.dma_map != NULL)
2729	ubsec_dma_free(sc, &me->me_q.q_mcr);
2730	if (me->me_q.q_ctx.dma_map != NULL) {
2731	memset(me->me_q.q_ctx.dma_vaddr, `0`,
2732	me->me_q.q_ctx.dma_size);
2733	ubsec_dma_free(sc, &me->me_q.q_ctx);
2734	}
2735	if (me->me_M.dma_map != NULL) {
2736	memset(me->me_M.dma_vaddr, `0`, me->me_M.dma_size);
2737	ubsec_dma_free(sc, &me->me_M);
2738	}
2739	if (me->me_E.dma_map != NULL) {
2740	memset(me->me_E.dma_vaddr, `0`, me->me_E.dma_size);
2741	ubsec_dma_free(sc, &me->me_E);
2742	}
2743	if (me->me_C.dma_map != NULL) {
2744	memset(me->me_C.dma_vaddr, `0`, me->me_C.dma_size);
2745	ubsec_dma_free(sc, &me->me_C);
2746	}
2747	if (me->me_epb.dma_map != NULL)
2748	ubsec_dma_free(sc, &me->me_epb);
2749	free(me, M_DEVBUF);
2750	}
2751	krp->krp_status = err;
2752	crypto_kdone(krp);
2753	return (`0`);
2754	}
2755
2756	/*
2757	* Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (hw normalization)
2758	*/
2759	static int
2760	ubsec_kprocess_modexp_hw(struct ubsec_softc sc, struct* cryptkop *krp,
2761	int hint)
2762	{
2763	struct ubsec_q2_modexp *me;
2764	struct ubsec_mcr *mcr;
2765	struct ubsec_ctx_modexp *ctx;
2766	struct ubsec_pktbuf *epb;
2767	int err = `0`;
2768	u_int nbits, normbits, mbits, shiftbits, ebits;
2769
2770	me = (struct ubsec_q2_modexp )malloc(sizeof* *me, M_DEVBUF, M_NOWAIT);
2771	if (me == NULL) {
2772	err = ENOMEM;
2773	goto errout;
2774	}
2775	memset(me, `0`, sizeof *me);
2776	me->me_krp = krp;
2777	me->me_q.q_type = UBS_CTXOP_MODEXP;
2778
2779	nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]);
2780	if (nbits <= `512`)
2781	normbits = `512`;
2782	else if (nbits <= `768`)
2783	normbits = `768`;
2784	else if (nbits <= `1024`)
2785	normbits = `1024`;
2786	else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= `1536`)
2787	normbits = `1536`;
2788	else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= `2048`)
2789	normbits = `2048`;
2790	else {
2791	err = E2BIG;
2792	goto errout;
2793	}
2794
2795	shiftbits = normbits - nbits;
2796
2797	/ XXX ??? /
2798	me->me_modbits = nbits;
2799	me->me_shiftbits = shiftbits;
2800	me->me_normbits = normbits;
2801
2802	/ Sanity check: result bits must be >= true modulus bits. /
2803	if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) {
2804	err = ERANGE;
2805	goto errout;
2806	}
2807
2808	if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
2809	&me->me_q.q_mcr, `0`)) {
2810	err = ENOMEM;
2811	goto errout;
2812	}
2813	mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr;
2814
2815	if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp),
2816	&me->me_q.q_ctx, `0`)) {
2817	err = ENOMEM;
2818	goto errout;
2819	}
2820
2821	mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]);
2822	if (mbits > nbits) {
2823	err = E2BIG;
2824	goto errout;
2825	}
2826	if (ubsec_dma_malloc(sc, normbits / `8`, &me->me_M, `0`)) {
2827	err = ENOMEM;
2828	goto errout;
2829	}
2830	memset(me->me_M.dma_vaddr, `0`, normbits / `8`);
2831	bcopy(krp->krp_param[UBS_MODEXP_PAR_M].crp_p,
2832	me->me_M.dma_vaddr, (mbits + `7`) / `8`);
2833
2834	if (ubsec_dma_malloc(sc, normbits / `8`, &me->me_C, `0`)) {
2835	err = ENOMEM;
2836	goto errout;
2837	}
2838	memset(me->me_C.dma_vaddr, `0`, me->me_C.dma_size);
2839
2840	ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]);
2841	if (ebits > nbits) {
2842	err = E2BIG;
2843	goto errout;
2844	}
2845	if (ubsec_dma_malloc(sc, normbits / `8`, &me->me_E, `0`)) {
2846	err = ENOMEM;
2847	goto errout;
2848	}
2849	memset(me->me_E.dma_vaddr, `0`, normbits / `8`);
2850	bcopy(krp->krp_param[UBS_MODEXP_PAR_E].crp_p,
2851	me->me_E.dma_vaddr, (ebits + `7`) / `8`);
2852
2853	if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf),
2854	&me->me_epb, `0`)) {
2855	err = ENOMEM;
2856	goto errout;
2857	}
2858	epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr;
2859	epb->pb_addr = htole32(me->me_E.dma_paddr);
2860	epb->pb_next = `0`;
2861	epb->pb_len = htole32((ebits + `7`) / `8`);
2862
2863	#ifdef UBSEC_DEBUG
2864	if (ubsec_debug) {
2865	printf("Epb ");
2866	ubsec_dump_pb(epb);
2867	}
2868	#endif
2869
2870	mcr->mcr_pkts = htole16(`1`);
2871	mcr->mcr_flags = `0`;
2872	mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr);
2873	mcr->mcr_reserved = `0`;
2874	mcr->mcr_pktlen = `0`;
2875
2876	mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr);
2877	mcr->mcr_ipktbuf.pb_len = htole32(normbits / `8`);
2878	mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr);
2879
2880	mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr);
2881	mcr->mcr_opktbuf.pb_next = `0`;
2882	mcr->mcr_opktbuf.pb_len = htole32(normbits / `8`);
2883
2884	#ifdef DIAGNOSTIC
2885	/ Misaligned output buffer will hang the chip. /
2886	if ((letoh32(mcr->mcr_opktbuf.pb_addr) & `3`) != `0`)
2887	panic("%s: modexp invalid addr 0x%x", device_xname(sc->sc_dev),
2888	letoh32(mcr->mcr_opktbuf.pb_addr));
2889	if ((letoh32(mcr->mcr_opktbuf.pb_len) & `3`) != `0`)
2890	panic("%s: modexp invalid len 0x%x", device_xname(sc->sc_dev),
2891	letoh32(mcr->mcr_opktbuf.pb_len));
2892	#endif
2893
2894	ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr;
2895	memset(ctx, `0`, sizeof(*ctx));
2896	memcpy(ctx->me_N, krp->krp_param[UBS_MODEXP_PAR_N].crp_p,
2897	(nbits + `7`) / `8`);
2898	ctx->me_len = htole16((normbits / `8`) + (`4` * sizeof(u_int16_t)));
2899	ctx->me_op = htole16(UBS_CTXOP_MODEXP);
2900	ctx->me_E_len = htole16(ebits);
2901	ctx->me_N_len = htole16(nbits);
2902
2903	#ifdef UBSEC_DEBUG
2904	if (ubsec_debug) {
2905	ubsec_dump_mcr(mcr);
2906	ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx);
2907	}
2908	#endif
2909
2910	/*
2911	* ubsec_feed2 will sync mcr and ctx, we just need to sync
2912	* everything else.
2913	*/
2914	bus_dmamap_sync(sc->sc_dmat, me->me_M.dma_map,
2915	`0`, me->me_M.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
2916	bus_dmamap_sync(sc->sc_dmat, me->me_E.dma_map,
2917	`0`, me->me_E.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
2918	bus_dmamap_sync(sc->sc_dmat, me->me_C.dma_map,
2919	`0`, me->me_C.dma_map->dm_mapsize, BUS_DMASYNC_PREREAD);
2920	bus_dmamap_sync(sc->sc_dmat, me->me_epb.dma_map,
2921	`0`, me->me_epb.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
2922
2923	/ Enqueue and we're done... /
2924	mutex_spin_enter(&sc->sc_mtx);
2925	SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next);
2926	ubsec_feed2(sc);
2927	mutex_spin_exit(&sc->sc_mtx);
2928
2929	return (`0`);
2930
2931	errout:
2932	if (me != NULL) {
2933	if (me->me_q.q_mcr.dma_map != NULL)
2934	ubsec_dma_free(sc, &me->me_q.q_mcr);
2935	if (me->me_q.q_ctx.dma_map != NULL) {
2936	memset(me->me_q.q_ctx.dma_vaddr, `0`,
2937	me->me_q.q_ctx.dma_size);
2938	ubsec_dma_free(sc, &me->me_q.q_ctx);
2939	}
2940	if (me->me_M.dma_map != NULL) {
2941	memset(me->me_M.dma_vaddr, `0`, me->me_M.dma_size);
2942	ubsec_dma_free(sc, &me->me_M);
2943	}
2944	if (me->me_E.dma_map != NULL) {
2945	memset(me->me_E.dma_vaddr, `0`, me->me_E.dma_size);
2946	ubsec_dma_free(sc, &me->me_E);
2947	}
2948	if (me->me_C.dma_map != NULL) {
2949	memset(me->me_C.dma_vaddr, `0`, me->me_C.dma_size);
2950	ubsec_dma_free(sc, &me->me_C);
2951	}
2952	if (me->me_epb.dma_map != NULL)
2953	ubsec_dma_free(sc, &me->me_epb);
2954	free(me, M_DEVBUF);
2955	}
2956	krp->krp_status = err;
2957	crypto_kdone(krp);
2958	return (`0`);
2959	}
2960
2961	static int
2962	ubsec_kprocess_rsapriv(struct ubsec_softc sc, struct* cryptkop *krp,
2963	int hint)
2964	{
2965	struct ubsec_q2_rsapriv *rp = NULL;
2966	struct ubsec_mcr *mcr;
2967	struct ubsec_ctx_rsapriv *ctx;
2968	int err = `0`;
2969	u_int padlen, msglen;
2970
2971	msglen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_P]);
2972	padlen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_Q]);
2973	if (msglen > padlen)
2974	padlen = msglen;
2975
2976	if (padlen <= `256`)
2977	padlen = `256`;
2978	else if (padlen <= `384`)
2979	padlen = `384`;
2980	else if (padlen <= `512`)
2981	padlen = `512`;
2982	else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= `768`)
2983	padlen = `768`;
2984	else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= `1024`)
2985	padlen = `1024`;
2986	else {
2987	err = E2BIG;
2988	goto errout;
2989	}
2990
2991	if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DP]) > padlen) {
2992	err = E2BIG;
2993	goto errout;
2994	}
2995
2996	if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DQ]) > padlen) {
2997	err = E2BIG;
2998	goto errout;
2999	}
3000
3001	if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_PINV]) > padlen) {
3002	err = E2BIG;
3003	goto errout;
3004	}
3005
3006	rp = malloc(sizeof *rp, M_DEVBUF, M_NOWAIT\|M_ZERO);
3007	if (rp == NULL)
3008	return (ENOMEM);
3009	rp->rpr_krp = krp;
3010	rp->rpr_q.q_type = UBS_CTXOP_RSAPRIV;
3011
3012	if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
3013	&rp->rpr_q.q_mcr, `0`)) {
3014	err = ENOMEM;
3015	goto errout;
3016	}
3017	mcr = (struct ubsec_mcr *)rp->rpr_q.q_mcr.dma_vaddr;
3018
3019	if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rsapriv),
3020	&rp->rpr_q.q_ctx, `0`)) {
3021	err = ENOMEM;
3022	goto errout;
3023	}
3024	ctx = (struct ubsec_ctx_rsapriv *)rp->rpr_q.q_ctx.dma_vaddr;
3025	memset(ctx, `0`, sizeof *ctx);
3026
3027	/ Copy in p /
3028	bcopy(krp->krp_param[UBS_RSAPRIV_PAR_P].crp_p,
3029	&ctx->rpr_buf[`0` * (padlen / `8`)],
3030	(krp->krp_param[UBS_RSAPRIV_PAR_P].crp_nbits + `7`) / `8`);
3031
3032	/ Copy in q /
3033	bcopy(krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_p,
3034	&ctx->rpr_buf[`1` * (padlen / `8`)],
3035	(krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_nbits + `7`) / `8`);
3036
3037	/ Copy in dp /
3038	bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_p,
3039	&ctx->rpr_buf[`2` * (padlen / `8`)],
3040	(krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_nbits + `7`) / `8`);
3041
3042	/ Copy in dq /
3043	bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_p,
3044	&ctx->rpr_buf[`3` * (padlen / `8`)],
3045	(krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_nbits + `7`) / `8`);
3046
3047	/ Copy in pinv /
3048	bcopy(krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_p,
3049	&ctx->rpr_buf[`4` * (padlen / `8`)],
3050	(krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_nbits + `7`) / `8`);
3051
3052	msglen = padlen * `2`;
3053
3054	/ Copy in input message (aligned buffer/length). /
3055	if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGIN]) > msglen) {
3056	/ Is this likely? /
3057	err = E2BIG;
3058	goto errout;
3059	}
3060	if (ubsec_dma_malloc(sc, (msglen + `7`) / `8`, &rp->rpr_msgin, `0`)) {
3061	err = ENOMEM;
3062	goto errout;
3063	}
3064	memset(rp->rpr_msgin.dma_vaddr, `0`, (msglen + `7`) / `8`);
3065	bcopy(krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_p,
3066	rp->rpr_msgin.dma_vaddr,
3067	(krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_nbits + `7`) / `8`);
3068
3069	/ Prepare space for output message (aligned buffer/length). /
3070	if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT]) < msglen) {
3071	/ Is this likely? /
3072	err = E2BIG;
3073	goto errout;
3074	}
3075	if (ubsec_dma_malloc(sc, (msglen + `7`) / `8`, &rp->rpr_msgout, `0`)) {
3076	err = ENOMEM;
3077	goto errout;
3078	}
3079	memset(rp->rpr_msgout.dma_vaddr, `0`, (msglen + `7`) / `8`);
3080
3081	mcr->mcr_pkts = htole16(`1`);
3082	mcr->mcr_flags = `0`;
3083	mcr->mcr_cmdctxp = htole32(rp->rpr_q.q_ctx.dma_paddr);
3084	mcr->mcr_ipktbuf.pb_addr = htole32(rp->rpr_msgin.dma_paddr);
3085	mcr->mcr_ipktbuf.pb_next = `0`;
3086	mcr->mcr_ipktbuf.pb_len = htole32(rp->rpr_msgin.dma_size);
3087	mcr->mcr_reserved = `0`;
3088	mcr->mcr_pktlen = htole16(msglen);
3089	mcr->mcr_opktbuf.pb_addr = htole32(rp->rpr_msgout.dma_paddr);
3090	mcr->mcr_opktbuf.pb_next = `0`;
3091	mcr->mcr_opktbuf.pb_len = htole32(rp->rpr_msgout.dma_size);
3092
3093	#ifdef DIAGNOSTIC
3094	if (rp->rpr_msgin.dma_paddr & `3` \|\| rp->rpr_msgin.dma_size & `3`) {
3095	panic("%s: rsapriv: invalid msgin 0x%lx(0x%lx)",
3096	device_xname(sc->sc_dev), (u_long) rp->rpr_msgin.dma_paddr,
3097	(u_long) rp->rpr_msgin.dma_size);
3098	}
3099	if (rp->rpr_msgout.dma_paddr & `3` \|\| rp->rpr_msgout.dma_size & `3`) {
3100	panic("%s: rsapriv: invalid msgout 0x%lx(0x%lx)",
3101	device_xname(sc->sc_dev), (u_long) rp->rpr_msgout.dma_paddr,
3102	(u_long) rp->rpr_msgout.dma_size);
3103	}
3104	#endif
3105
3106	ctx->rpr_len = (sizeof(u_int16_t) * `4`) + (`5` * (padlen / `8`));
3107	ctx->rpr_op = htole16(UBS_CTXOP_RSAPRIV);
3108	ctx->rpr_q_len = htole16(padlen);
3109	ctx->rpr_p_len = htole16(padlen);
3110
3111	/*
3112	* ubsec_feed2 will sync mcr and ctx, we just need to sync
3113	* everything else.
3114	*/
3115	bus_dmamap_sync(sc->sc_dmat, rp->rpr_msgin.dma_map,
3116	`0`, rp->rpr_msgin.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
3117	bus_dmamap_sync(sc->sc_dmat, rp->rpr_msgout.dma_map,
3118	`0`, rp->rpr_msgout.dma_map->dm_mapsize, BUS_DMASYNC_PREREAD);
3119
3120	/ Enqueue and we're done... /
3121	mutex_spin_enter(&sc->sc_mtx);
3122	SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rp->rpr_q, q_next);
3123	ubsec_feed2(sc);
3124	ubsecstats.hst_modexpcrt++;
3125	mutex_spin_exit(&sc->sc_mtx);
3126	return (`0`);
3127
3128	errout:
3129	if (rp != NULL) {
3130	if (rp->rpr_q.q_mcr.dma_map != NULL)
3131	ubsec_dma_free(sc, &rp->rpr_q.q_mcr);
3132	if (rp->rpr_msgin.dma_map != NULL) {
3133	memset(rp->rpr_msgin.dma_vaddr, `0`,
3134	rp->rpr_msgin.dma_size);
3135	ubsec_dma_free(sc, &rp->rpr_msgin);
3136	}
3137	if (rp->rpr_msgout.dma_map != NULL) {
3138	memset(rp->rpr_msgout.dma_vaddr, `0`,
3139	rp->rpr_msgout.dma_size);
3140	ubsec_dma_free(sc, &rp->rpr_msgout);
3141	}
3142	free(rp, M_DEVBUF);
3143	}
3144	krp->krp_status = err;
3145	crypto_kdone(krp);
3146	return (`0`);
3147	}
3148
3149	#ifdef UBSEC_DEBUG
3150	static void
3151	ubsec_dump_pb(volatile struct ubsec_pktbuf *pb)
3152	{
3153	printf("addr 0x%x (0x%x) next 0x%x\n",
3154	pb->pb_addr, pb->pb_len, pb->pb_next);
3155	}
3156
3157	static void
3158	ubsec_dump_ctx2(volatile struct ubsec_ctx_keyop *c)
3159	{
3160	printf("CTX (0x%x):\n", c->ctx_len);
3161	switch (letoh16(c->ctx_op)) {
3162	case UBS_CTXOP_RNGBYPASS:
3163	case UBS_CTXOP_RNGSHA1:
3164	break;
3165	case UBS_CTXOP_MODEXP:
3166	{
3167	struct ubsec_ctx_modexp cx = (void* *)c;
3168	int i, len;
3169
3170	printf(" Elen %u, Nlen %u\n",
3171	letoh16(cx->me_E_len), letoh16(cx->me_N_len));
3172	len = (cx->me_N_len + `7`)/`8`;
3173	for (i = `0`; i < len; i++)
3174	printf("%s%02x", (i == `0`) ? " N: " : ":", cx->me_N[i]);
3175	printf("\n");
3176	break;
3177	}
3178	default:
3179	printf("unknown context: %x\n", c->ctx_op);
3180	}
3181	printf("END CTX\n");
3182	}
3183
3184	static void
3185	ubsec_dump_mcr(struct ubsec_mcr *mcr)
3186	{
3187	volatile struct ubsec_mcr_add *ma;
3188	int i;
3189
3190	printf("MCR:\n");
3191	printf(" pkts: %u, flags 0x%x\n",
3192	letoh16(mcr->mcr_pkts), letoh16(mcr->mcr_flags));
3193	ma = (volatile struct ubsec_mcr_add *)&mcr->mcr_cmdctxp;
3194	for (i = `0`; i < letoh16(mcr->mcr_pkts); i++) {
3195	printf(" %d: ctx 0x%x len 0x%x rsvd 0x%x\n", i,
3196	letoh32(ma->mcr_cmdctxp), letoh16(ma->mcr_pktlen),
3197	letoh16(ma->mcr_reserved));
3198	printf(" %d: ipkt ", i);
3199	ubsec_dump_pb(&ma->mcr_ipktbuf);
3200	printf(" %d: opkt ", i);
3201	ubsec_dump_pb(&ma->mcr_opktbuf);
3202	ma++;
3203	}
3204	printf("END MCR\n");
3205	}
3206	#endif /* UBSEC_DEBUG */
3207
3208	/*
3209	* Return the number of significant bits of a big number.
3210	*/
3211	static int
3212	ubsec_ksigbits(struct crparam *cr)
3213	{
3214	u_int plen = (cr->crp_nbits + `7`) / `8`;
3215	int i, sig = plen * `8`;
3216	u_int8_t c, *p = cr->crp_p;
3217
3218	for (i = plen - `1`; i >= `0`; i--) {
3219	c = p[i];
3220	if (c != `0`) {
3221	while ((c & `0x80`) == `0`) {
3222	sig--;
3223	c <<= `1`;
3224	}
3225	break;
3226	}
3227	sig -= `8`;
3228	}
3229	return (sig);
3230	}
3231
3232	static void
3233	ubsec_kshift_r(u_int shiftbits, u_int8_t *src, u_int srcbits,
3234	u_int8_t *dst, u_int dstbits)
3235	{
3236	u_int slen, dlen;
3237	int i, si, di, n;
3238
3239	slen = (srcbits + `7`) / `8`;
3240	dlen = (dstbits + `7`) / `8`;
3241
3242	for (i = `0`; i < slen; i++)
3243	dst[i] = src[i];
3244	for (i = `0`; i < dlen - slen; i++)
3245	dst[slen + i] = `0`;
3246
3247	n = shiftbits / `8`;
3248	if (n != `0`) {
3249	si = dlen - n - `1`;
3250	di = dlen - `1`;
3251	while (si >= `0`)
3252	dst[di--] = dst[si--];
3253	while (di >= `0`)
3254	dst[di--] = `0`;
3255	}
3256
3257	n = shiftbits % `8`;
3258	if (n != `0`) {
3259	for (i = dlen - `1`; i > `0`; i--)
3260	dst[i] = (dst[i] << n) \|
3261	(dst[i - `1`] >> (`8` - n));
3262	dst[`0`] = dst[`0`] << n;
3263	}
3264	}
3265
3266	static void
3267	ubsec_kshift_l(u_int shiftbits, u_int8_t *src, u_int srcbits,
3268	u_int8_t *dst, u_int dstbits)
3269	{
3270	int slen, dlen, i, n;
3271
3272	slen = (srcbits + `7`) / `8`;
3273	dlen = (dstbits + `7`) / `8`;
3274
3275	n = shiftbits / `8`;
3276	for (i = `0`; i < slen; i++)
3277	dst[i] = src[i + n];
3278	for (i = `0`; i < dlen - slen; i++)
3279	dst[slen + i] = `0`;
3280
3281	n = shiftbits % `8`;
3282	if (n != `0`) {
3283	for (i = `0`; i < (dlen - `1`); i++)
3284	dst[i] = (dst[i] >> n) \| (dst[i + `1`] << (`8` - n));
3285	dst[dlen - `1`] = dst[dlen - `1`] >> n;
3286	}
3287	}
3288

Browse the source code of src/src/sys/dev/pci/ubsec.c