1/* $NetBSD: bus_space.c,v 1.38 2012/01/27 18:53:07 para Exp $ */
2
3/*-
4 * Copyright (c) 1996, 1997, 1998 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Charles M. Hannum and by Jason R. Thorpe of the Numerical Aerospace
9 * Simulation Facility, NASA Ames Research Center.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 * POSSIBILITY OF SUCH DAMAGE.
31 */
32
33#include <sys/cdefs.h>
34__KERNEL_RCSID(0, "$NetBSD: bus_space.c,v 1.38 2012/01/27 18:53:07 para Exp $");
35
36#include <sys/param.h>
37#include <sys/systm.h>
38#include <sys/malloc.h>
39#include <sys/extent.h>
40#include <sys/kmem.h>
41
42#include <uvm/uvm_extern.h>
43
44#include <dev/isa/isareg.h>
45
46#include <sys/bus.h>
47#include <machine/pio.h>
48#include <machine/isa_machdep.h>
49
50#ifdef XEN
51#include <xen/hypervisor.h>
52#endif
53
54/*
55 * Macros for sanity-checking the aligned-ness of pointers passed to
56 * bus space ops. These are not strictly necessary on the x86, but
57 * could lead to performance improvements, and help catch problems
58 * with drivers that would creep up on other architectures.
59 */
60#ifdef BUS_SPACE_DEBUG
61#define BUS_SPACE_ALIGNED_ADDRESS(p, t) \
62 ((((u_long)(p)) & (sizeof(t)-1)) == 0)
63
64#define BUS_SPACE_ADDRESS_SANITY(p, t, d) \
65({ \
66 if (BUS_SPACE_ALIGNED_ADDRESS((p), t) == 0) { \
67 printf("%s 0x%lx not aligned to %zu bytes %s:%d\n", \
68 d, (u_long)(p), sizeof(t), __FILE__, __LINE__); \
69 } \
70 (void) 0; \
71})
72#else
73#define BUS_SPACE_ADDRESS_SANITY(p,t,d) (void) 0
74#endif /* BUS_SPACE_DEBUG */
75
76/*
77 * Extent maps to manage I/O and memory space. Allocate
78 * storage for 8 regions in each, initially. Later, ioport_malloc_safe
79 * will indicate that it's safe to use malloc() to dynamically allocate
80 * region descriptors.
81 *
82 * N.B. At least two regions are _always_ allocated from the iomem
83 * extent map; (0 -> ISA hole) and (end of ISA hole -> end of RAM).
84 *
85 * The extent maps are not static! Machine-dependent ISA and EISA
86 * routines need access to them for bus address space allocation.
87 */
88static long ioport_ex_storage[EXTENT_FIXED_STORAGE_SIZE(16) / sizeof(long)];
89static long iomem_ex_storage[EXTENT_FIXED_STORAGE_SIZE(64) / sizeof(long)];
90struct extent *ioport_ex;
91struct extent *iomem_ex;
92static int ioport_malloc_safe;
93
94static struct bus_space_tag x86_io = { .bst_type = X86_BUS_SPACE_IO };
95static struct bus_space_tag x86_mem = { .bst_type = X86_BUS_SPACE_MEM };
96
97bus_space_tag_t x86_bus_space_io = &x86_io;
98bus_space_tag_t x86_bus_space_mem = &x86_mem;
99
100int x86_mem_add_mapping(bus_addr_t, bus_size_t,
101 int, bus_space_handle_t *);
102
103static inline bool
104x86_bus_space_is_io(bus_space_tag_t t)
105{
106 return t->bst_type == X86_BUS_SPACE_IO;
107}
108
109static inline bool
110x86_bus_space_is_mem(bus_space_tag_t t)
111{
112 return t->bst_type == X86_BUS_SPACE_MEM;
113}
114
115void
116x86_bus_space_init(void)
117{
118 /*
119 * Initialize the I/O port and I/O mem extent maps.
120 * Note: we don't have to check the return value since
121 * creation of a fixed extent map will never fail (since
122 * descriptor storage has already been allocated).
123 *
124 * N.B. The iomem extent manages _all_ physical addresses
125 * on the machine. When the amount of RAM is found, the two
126 * extents of RAM are allocated from the map (0 -> ISA hole
127 * and end of ISA hole -> end of RAM).
128 */
129 ioport_ex = extent_create("ioport", 0x0, 0xffff,
130 (void *)ioport_ex_storage, sizeof(ioport_ex_storage),
131 EX_NOCOALESCE|EX_NOWAIT);
132 iomem_ex = extent_create("iomem", 0x0, 0xffffffff,
133 (void *)iomem_ex_storage, sizeof(iomem_ex_storage),
134 EX_NOCOALESCE|EX_NOWAIT);
135
136#ifdef XEN
137 /* We are privileged guest os - should have IO privileges. */
138 if (xendomain_is_privileged()) {
139 struct physdev_op physop;
140 physop.cmd = PHYSDEVOP_SET_IOPL;
141 physop.u.set_iopl.iopl = 1;
142 if (HYPERVISOR_physdev_op(&physop) != 0)
143 panic("Unable to obtain IOPL, "
144 "despite being SIF_PRIVILEGED");
145 }
146#endif /* XEN */
147}
148
149void
150x86_bus_space_mallocok(void)
151{
152
153 ioport_malloc_safe = 1;
154}
155
156int
157bus_space_map(bus_space_tag_t t, bus_addr_t bpa, bus_size_t size,
158 int flags, bus_space_handle_t *bshp)
159{
160 bus_space_reservation_t bsr;
161 bus_space_tag_t it;
162 int error;
163
164 if ((t->bst_exists & BUS_SPACE_OVERRIDE_MAP) == 0)
165 ; /* skip override */
166 else for (it = t; it != NULL; it = it->bst_super) {
167 if ((it->bst_present & BUS_SPACE_OVERRIDE_MAP) == 0)
168 continue;
169 return (*it->bst_ov->ov_space_map)(it->bst_ctx, t, bpa, size,
170 flags, bshp);
171 }
172
173 error = bus_space_reserve(t, bpa, size, flags, &bsr);
174 if (error != 0)
175 return error;
176
177 error = bus_space_reservation_map(t, &bsr, flags, bshp);
178 if (error != 0)
179 bus_space_release(t, &bsr);
180
181 return error;
182}
183
184int
185bus_space_reservation_map(bus_space_tag_t t, bus_space_reservation_t *bsr,
186 int flags, bus_space_handle_t *bshp)
187{
188 bus_addr_t bpa;
189 bus_size_t size;
190 bus_space_tag_t it;
191
192 if ((t->bst_exists & BUS_SPACE_OVERRIDE_RESERVATION_MAP) == 0)
193 ; /* skip override */
194 else for (it = t; it != NULL; it = it->bst_super) {
195 if ((it->bst_present & BUS_SPACE_OVERRIDE_RESERVATION_MAP) == 0)
196 continue;
197 return (*it->bst_ov->ov_space_reservation_map)(it->bst_ctx, t,
198 bsr, flags, bshp);
199 }
200
201 bpa = bus_space_reservation_addr(bsr);
202 size = bus_space_reservation_size(bsr);
203
204 /*
205 * For I/O space, that's all she wrote.
206 */
207 if (x86_bus_space_is_io(t)) {
208 *bshp = bpa;
209 return 0;
210 }
211
212#ifndef XEN
213 if (bpa >= IOM_BEGIN && (bpa + size) != 0 && (bpa + size) <= IOM_END) {
214 *bshp = (bus_space_handle_t)ISA_HOLE_VADDR(bpa);
215 return 0;
216 }
217#endif /* !XEN */
218
219 /*
220 * For memory space, map the bus physical address to
221 * a kernel virtual address.
222 */
223 return x86_mem_add_mapping(bpa, size, flags, bshp);
224}
225
226int
227_x86_memio_map(bus_space_tag_t t, bus_addr_t bpa, bus_size_t size,
228 int flags, bus_space_handle_t *bshp)
229{
230
231 /*
232 * For I/O space, just fill in the handle.
233 */
234 if (x86_bus_space_is_io(t)) {
235 if (flags & BUS_SPACE_MAP_LINEAR)
236 return (EOPNOTSUPP);
237 *bshp = bpa;
238 return (0);
239 }
240
241 /*
242 * For memory space, map the bus physical address to
243 * a kernel virtual address.
244 */
245 return x86_mem_add_mapping(bpa, size, flags, bshp);
246}
247
248int
249bus_space_reserve(bus_space_tag_t t,
250 bus_addr_t bpa,
251 bus_size_t size,
252 int flags, bus_space_reservation_t *bsrp)
253{
254 struct extent *ex;
255 int error;
256 bus_space_tag_t it;
257
258 if ((t->bst_exists & BUS_SPACE_OVERRIDE_RESERVE) == 0)
259 ; /* skip override */
260 else for (it = t; it != NULL; it = it->bst_super) {
261 if ((it->bst_present & BUS_SPACE_OVERRIDE_RESERVE) == 0)
262 continue;
263 return (*it->bst_ov->ov_space_reserve)(it->bst_ctx, t,
264 bpa, size, flags, bsrp);
265 }
266
267 /*
268 * Pick the appropriate extent map.
269 */
270 if (x86_bus_space_is_io(t)) {
271 if (flags & BUS_SPACE_MAP_LINEAR)
272 return (EOPNOTSUPP);
273 ex = ioport_ex;
274 } else if (x86_bus_space_is_mem(t))
275 ex = iomem_ex;
276 else
277 panic("x86_memio_alloc: bad bus space tag");
278
279 /*
280 * Before we go any further, let's make sure that this
281 * region is available.
282 */
283 error = extent_alloc_region(ex, bpa, size,
284 EX_NOWAIT | (ioport_malloc_safe ? EX_MALLOCOK : 0));
285
286 if (error != 0)
287 return error;
288
289 bus_space_reservation_init(bsrp, bpa, size);
290
291 return 0;
292}
293
294int
295bus_space_reserve_subregion(bus_space_tag_t t,
296 bus_addr_t rstart, bus_addr_t rend,
297 const bus_size_t size, const bus_size_t alignment,
298 const bus_size_t boundary,
299 const int flags, bus_space_reservation_t *bsrp)
300{
301 bus_space_reservation_t bsr;
302 struct extent *ex;
303 u_long bpa;
304 int error;
305 bus_space_tag_t it;
306
307 if ((t->bst_exists & BUS_SPACE_OVERRIDE_RESERVE_SUBREGION) == 0)
308 ; /* skip override */
309 else for (it = t; it != NULL; it = it->bst_super) {
310 if ((it->bst_present & BUS_SPACE_OVERRIDE_RESERVE_SUBREGION) ==
311 0)
312 continue;
313 return (*it->bst_ov->ov_space_reserve_subregion)(it->bst_ctx, t,
314 rstart, rend, size, alignment, boundary, flags, bsrp);
315 }
316
317 /*
318 * Pick the appropriate extent map.
319 */
320 if (x86_bus_space_is_io(t)) {
321 if (flags & BUS_SPACE_MAP_LINEAR)
322 return (EOPNOTSUPP);
323 ex = ioport_ex;
324 } else if (x86_bus_space_is_mem(t))
325 ex = iomem_ex;
326 else
327 panic("x86_memio_alloc: bad bus space tag");
328
329 /*
330 * Sanity check the allocation against the extent's boundaries.
331 */
332 rstart = MAX(rstart, ex->ex_start);
333 rend = MIN(rend, ex->ex_end);
334 if (rstart >= rend)
335 panic("x86_memio_alloc: bad region start/end");
336
337 /*
338 * Do the requested allocation.
339 */
340 error = extent_alloc_subregion(ex, rstart, rend, size, alignment,
341 boundary,
342 EX_FAST | EX_NOWAIT | (ioport_malloc_safe ? EX_MALLOCOK : 0),
343 &bpa);
344
345 if (error)
346 return (error);
347
348 bus_space_reservation_init(&bsr, bpa, size);
349
350 *bsrp = bsr;
351
352 return 0;
353}
354
355void
356bus_space_release(bus_space_tag_t t, bus_space_reservation_t *bsr)
357{
358 struct extent *ex;
359 bus_space_tag_t it;
360
361 if ((t->bst_exists & BUS_SPACE_OVERRIDE_RELEASE) == 0)
362 ; /* skip override */
363 else for (it = t; it != NULL; it = it->bst_super) {
364 if ((it->bst_present & BUS_SPACE_OVERRIDE_RELEASE) == 0)
365 continue;
366 (*it->bst_ov->ov_space_release)(it->bst_ctx, t, bsr);
367 return;
368 }
369
370 /*
371 * Pick the appropriate extent map.
372 */
373 if (x86_bus_space_is_io(t)) {
374 ex = ioport_ex;
375 } else if (x86_bus_space_is_mem(t))
376 ex = iomem_ex;
377 else
378 panic("x86_memio_alloc: bad bus space tag");
379
380 if (extent_free(ex, bus_space_reservation_addr(bsr),
381 bus_space_reservation_size(bsr), EX_NOWAIT |
382 (ioport_malloc_safe ? EX_MALLOCOK : 0))) {
383 printf("%s: pa 0x%jx, size 0x%jx\n", __func__,
384 (uintmax_t)bus_space_reservation_addr(bsr),
385 (uintmax_t)bus_space_reservation_size(bsr));
386 printf("%s: can't free region\n", __func__);
387 }
388}
389
390int
391bus_space_alloc(bus_space_tag_t t, bus_addr_t rstart, bus_addr_t rend,
392 bus_size_t size, bus_size_t alignment, bus_size_t boundary,
393 int flags, bus_addr_t *bpap, bus_space_handle_t *bshp)
394{
395 bus_space_reservation_t bsr;
396 bus_space_tag_t it;
397 int error;
398
399 if ((t->bst_exists & BUS_SPACE_OVERRIDE_ALLOC) == 0)
400 ; /* skip override */
401 else for (it = t; it != NULL; it = it->bst_super) {
402 if ((it->bst_present & BUS_SPACE_OVERRIDE_ALLOC) == 0)
403 continue;
404 return (*it->bst_ov->ov_space_alloc)(it->bst_ctx, t,
405 rstart, rend, size, alignment, boundary, flags, bpap, bshp);
406 }
407
408 /*
409 * Do the requested allocation.
410 */
411 error = bus_space_reserve_subregion(t, rstart, rend, size, alignment,
412 boundary, flags, &bsr);
413
414 if (error != 0)
415 return error;
416
417 error = bus_space_reservation_map(t, &bsr, flags, bshp);
418 if (error != 0)
419 bus_space_release(t, &bsr);
420
421 *bpap = bus_space_reservation_addr(&bsr);
422
423 return error;
424}
425
426int
427x86_mem_add_mapping(bus_addr_t bpa, bus_size_t size,
428 int flags, bus_space_handle_t *bshp)
429{
430 paddr_t pa, endpa;
431 vaddr_t va, sva;
432 u_int pmapflags;
433
434 pa = x86_trunc_page(bpa);
435 endpa = x86_round_page(bpa + size);
436
437 pmapflags = PMAP_NOCACHE;
438 if ((flags & BUS_SPACE_MAP_CACHEABLE) != 0)
439 pmapflags = 0;
440 else if (flags & BUS_SPACE_MAP_PREFETCHABLE)
441 pmapflags = PMAP_WRITE_COMBINE;
442
443#ifdef DIAGNOSTIC
444 if (endpa != 0 && endpa <= pa)
445 panic("x86_mem_add_mapping: overflow");
446#endif
447
448#ifdef XEN
449 if (bpa >= IOM_BEGIN && (bpa + size) != 0 && (bpa + size) <= IOM_END) {
450 sva = (vaddr_t)ISA_HOLE_VADDR(pa);
451 } else
452#endif /* XEN */
453 {
454 sva = uvm_km_alloc(kernel_map, endpa - pa, 0,
455 UVM_KMF_VAONLY | UVM_KMF_NOWAIT);
456 if (sva == 0)
457 return (ENOMEM);
458 }
459
460 *bshp = (bus_space_handle_t)(sva + (bpa & PGOFSET));
461
462 for (va = sva; pa != endpa; pa += PAGE_SIZE, va += PAGE_SIZE) {
463 pmap_kenter_ma(va, pa, VM_PROT_READ | VM_PROT_WRITE, pmapflags);
464 }
465 pmap_update(pmap_kernel());
466
467 return 0;
468}
469
470bool
471bus_space_is_equal(bus_space_tag_t t1, bus_space_tag_t t2)
472{
473 if (t1 == NULL || t2 == NULL)
474 return false;
475 return t1->bst_type == t2->bst_type;
476}
477
478/*
479 * void _x86_memio_unmap(bus_space_tag bst, bus_space_handle bsh,
480 * bus_size_t size, bus_addr_t *adrp)
481 *
482 * This function unmaps memory- or io-space mapped by the function
483 * _x86_memio_map(). This function works nearly as same as
484 * x86_memio_unmap(), but this function does not ask kernel
485 * built-in extents and returns physical address of the bus space,
486 * for the convenience of the extra extent manager.
487 */
488void
489_x86_memio_unmap(bus_space_tag_t t, bus_space_handle_t bsh,
490 bus_size_t size, bus_addr_t *adrp)
491{
492 u_long va, endva;
493 bus_addr_t bpa;
494
495 /*
496 * Find the correct extent and bus physical address.
497 */
498 if (x86_bus_space_is_io(t)) {
499 bpa = bsh;
500 } else if (x86_bus_space_is_mem(t)) {
501 if (bsh >= atdevbase && (bsh + size) != 0 &&
502 (bsh + size) <= (atdevbase + IOM_SIZE)) {
503 bpa = (bus_addr_t)ISA_PHYSADDR(bsh);
504 } else {
505
506 va = x86_trunc_page(bsh);
507 endva = x86_round_page(bsh + size);
508
509#ifdef DIAGNOSTIC
510 if (endva <= va) {
511 panic("_x86_memio_unmap: overflow");
512 }
513#endif
514
515 if (pmap_extract_ma(pmap_kernel(), va, &bpa) == FALSE) {
516 panic("_x86_memio_unmap:"
517 " wrong virtual address");
518 }
519 bpa += (bsh & PGOFSET);
520 pmap_kremove(va, endva - va);
521 pmap_update(pmap_kernel());
522
523 /*
524 * Free the kernel virtual mapping.
525 */
526 uvm_km_free(kernel_map, va, endva - va, UVM_KMF_VAONLY);
527 }
528 } else {
529 panic("_x86_memio_unmap: bad bus space tag");
530 }
531
532 if (adrp != NULL) {
533 *adrp = bpa;
534 }
535}
536
537static void
538bus_space_reservation_unmap1(bus_space_tag_t t, const bus_space_handle_t bsh,
539 const bus_size_t size, bus_addr_t *bpap)
540{
541 u_long va, endva;
542 bus_addr_t bpa;
543
544 /*
545 * Find the correct extent and bus physical address.
546 */
547 if (x86_bus_space_is_io(t)) {
548 bpa = bsh;
549 } else if (x86_bus_space_is_mem(t)) {
550 if (bsh >= atdevbase && (bsh + size) != 0 &&
551 (bsh + size) <= (atdevbase + IOM_SIZE)) {
552 bpa = (bus_addr_t)ISA_PHYSADDR(bsh);
553 goto ok;
554 }
555
556 va = x86_trunc_page(bsh);
557 endva = x86_round_page(bsh + size);
558
559#ifdef DIAGNOSTIC
560 if (endva <= va)
561 panic("x86_memio_unmap: overflow");
562#endif
563
564 (void) pmap_extract_ma(pmap_kernel(), va, &bpa);
565 bpa += (bsh & PGOFSET);
566
567 pmap_kremove(va, endva - va);
568 pmap_update(pmap_kernel());
569
570 /*
571 * Free the kernel virtual mapping.
572 */
573 uvm_km_free(kernel_map, va, endva - va, UVM_KMF_VAONLY);
574 } else
575 panic("x86_memio_unmap: bad bus space tag");
576ok:
577 if (bpap != NULL)
578 *bpap = bpa;
579}
580
581void
582bus_space_reservation_unmap(bus_space_tag_t t, const bus_space_handle_t bsh,
583 const bus_size_t size)
584{
585 bus_space_tag_t it;
586
587 if ((t->bst_exists & BUS_SPACE_OVERRIDE_RESERVATION_UNMAP) == 0)
588 ; /* skip override */
589 else for (it = t; it != NULL; it = it->bst_super) {
590 if ((it->bst_present & BUS_SPACE_OVERRIDE_RESERVATION_UNMAP) ==
591 0)
592 continue;
593 (*it->bst_ov->ov_space_reservation_unmap)(it->bst_ctx,
594 t, bsh, size);
595 return;
596 }
597
598 bus_space_reservation_unmap1(t, bsh, size, NULL);
599}
600
601void
602bus_space_unmap(bus_space_tag_t t, const bus_space_handle_t bsh,
603 const bus_size_t size)
604{
605 bus_addr_t addr;
606 bus_space_reservation_t bsr;
607 bus_space_tag_t it;
608
609 if ((t->bst_exists & BUS_SPACE_OVERRIDE_UNMAP) == 0)
610 ; /* skip override */
611 else for (it = t; it != NULL; it = it->bst_super) {
612 if ((it->bst_present & BUS_SPACE_OVERRIDE_UNMAP) == 0)
613 continue;
614 (*it->bst_ov->ov_space_unmap)(it->bst_ctx, t, bsh, size);
615 return;
616 }
617
618 bus_space_reservation_unmap1(t, bsh, size, &addr);
619
620 bus_space_reservation_init(&bsr, addr, size);
621 bus_space_release(t, &bsr);
622}
623
624void
625bus_space_free(bus_space_tag_t t, bus_space_handle_t bsh, bus_size_t size)
626{
627 bus_space_tag_t it;
628
629 if ((t->bst_exists & BUS_SPACE_OVERRIDE_FREE) == 0)
630 ; /* skip override */
631 else for (it = t; it != NULL; it = it->bst_super) {
632 if ((it->bst_present & BUS_SPACE_OVERRIDE_FREE) == 0)
633 continue;
634 (*it->bst_ov->ov_space_free)(it->bst_ctx, t, bsh, size);
635 return;
636 }
637 /* bus_space_unmap() does all that we need to do. */
638 bus_space_unmap(t, bsh, size);
639}
640
641int
642bus_space_subregion(bus_space_tag_t t, bus_space_handle_t bsh,
643 bus_size_t offset, bus_size_t size, bus_space_handle_t *nbshp)
644{
645
646 *nbshp = bsh + offset;
647 return (0);
648}
649
650paddr_t
651bus_space_mmap(bus_space_tag_t t, bus_addr_t addr, off_t off, int prot,
652 int flags)
653{
654 paddr_t pflags = 0;
655
656 /* Can't mmap I/O space. */
657 if (x86_bus_space_is_io(t))
658 return (-1);
659
660 /*
661 * "addr" is the base address of the device we're mapping.
662 * "off" is the offset into that device.
663 *
664 * Note we are called for each "page" in the device that
665 * the upper layers want to map.
666 */
667 if (flags & BUS_SPACE_MAP_PREFETCHABLE)
668 pflags |= X86_MMAP_FLAG_PREFETCH;
669
670 return x86_btop(addr + off) | (pflags << X86_MMAP_FLAG_SHIFT);
671}
672
673void
674bus_space_set_multi_1(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
675 uint8_t v, size_t c)
676{
677 vaddr_t addr = h + o;
678
679 if (x86_bus_space_is_io(t))
680 while (c--)
681 outb(addr, v);
682 else
683 while (c--)
684 *(volatile uint8_t *)(addr) = v;
685}
686
687void
688bus_space_set_multi_2(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
689 uint16_t v, size_t c)
690{
691 vaddr_t addr = h + o;
692
693 BUS_SPACE_ADDRESS_SANITY(addr, uint16_t, "bus addr");
694
695 if (x86_bus_space_is_io(t))
696 while (c--)
697 outw(addr, v);
698 else
699 while (c--)
700 *(volatile uint16_t *)(addr) = v;
701}
702
703void
704bus_space_set_multi_4(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
705 uint32_t v, size_t c)
706{
707 vaddr_t addr = h + o;
708
709 BUS_SPACE_ADDRESS_SANITY(addr, uint32_t, "bus addr");
710
711 if (x86_bus_space_is_io(t))
712 while (c--)
713 outl(addr, v);
714 else
715 while (c--)
716 *(volatile uint32_t *)(addr) = v;
717}
718
719void
720bus_space_set_region_1(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
721 uint8_t v, size_t c)
722{
723 vaddr_t addr = h + o;
724
725 if (x86_bus_space_is_io(t))
726 for (; c != 0; c--, addr++)
727 outb(addr, v);
728 else
729 for (; c != 0; c--, addr++)
730 *(volatile uint8_t *)(addr) = v;
731}
732
733void
734bus_space_set_region_2(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
735 uint16_t v, size_t c)
736{
737 vaddr_t addr = h + o;
738
739 BUS_SPACE_ADDRESS_SANITY(addr, uint16_t, "bus addr");
740
741 if (x86_bus_space_is_io(t))
742 for (; c != 0; c--, addr += 2)
743 outw(addr, v);
744 else
745 for (; c != 0; c--, addr += 2)
746 *(volatile uint16_t *)(addr) = v;
747}
748
749void
750bus_space_set_region_4(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
751 uint32_t v, size_t c)
752{
753 vaddr_t addr = h + o;
754
755 BUS_SPACE_ADDRESS_SANITY(addr, uint32_t, "bus addr");
756
757 if (x86_bus_space_is_io(t))
758 for (; c != 0; c--, addr += 4)
759 outl(addr, v);
760 else
761 for (; c != 0; c--, addr += 4)
762 *(volatile uint32_t *)(addr) = v;
763}
764
765void
766bus_space_copy_region_1(bus_space_tag_t t, bus_space_handle_t h1,
767 bus_size_t o1, bus_space_handle_t h2,
768 bus_size_t o2, size_t c)
769{
770 vaddr_t addr1 = h1 + o1;
771 vaddr_t addr2 = h2 + o2;
772
773 if (x86_bus_space_is_io(t)) {
774 if (addr1 >= addr2) {
775 /* src after dest: copy forward */
776 for (; c != 0; c--, addr1++, addr2++)
777 outb(addr2, inb(addr1));
778 } else {
779 /* dest after src: copy backwards */
780 for (addr1 += (c - 1), addr2 += (c - 1);
781 c != 0; c--, addr1--, addr2--)
782 outb(addr2, inb(addr1));
783 }
784 } else {
785 if (addr1 >= addr2) {
786 /* src after dest: copy forward */
787 for (; c != 0; c--, addr1++, addr2++)
788 *(volatile uint8_t *)(addr2) =
789 *(volatile uint8_t *)(addr1);
790 } else {
791 /* dest after src: copy backwards */
792 for (addr1 += (c - 1), addr2 += (c - 1);
793 c != 0; c--, addr1--, addr2--)
794 *(volatile uint8_t *)(addr2) =
795 *(volatile uint8_t *)(addr1);
796 }
797 }
798}
799
800void
801bus_space_copy_region_2(bus_space_tag_t t, bus_space_handle_t h1,
802 bus_size_t o1, bus_space_handle_t h2,
803 bus_size_t o2, size_t c)
804{
805 vaddr_t addr1 = h1 + o1;
806 vaddr_t addr2 = h2 + o2;
807
808 BUS_SPACE_ADDRESS_SANITY(addr1, uint16_t, "bus addr 1");
809 BUS_SPACE_ADDRESS_SANITY(addr2, uint16_t, "bus addr 2");
810
811 if (x86_bus_space_is_io(t)) {
812 if (addr1 >= addr2) {
813 /* src after dest: copy forward */
814 for (; c != 0; c--, addr1 += 2, addr2 += 2)
815 outw(addr2, inw(addr1));
816 } else {
817 /* dest after src: copy backwards */
818 for (addr1 += 2 * (c - 1), addr2 += 2 * (c - 1);
819 c != 0; c--, addr1 -= 2, addr2 -= 2)
820 outw(addr2, inw(addr1));
821 }
822 } else {
823 if (addr1 >= addr2) {
824 /* src after dest: copy forward */
825 for (; c != 0; c--, addr1 += 2, addr2 += 2)
826 *(volatile uint16_t *)(addr2) =
827 *(volatile uint16_t *)(addr1);
828 } else {
829 /* dest after src: copy backwards */
830 for (addr1 += 2 * (c - 1), addr2 += 2 * (c - 1);
831 c != 0; c--, addr1 -= 2, addr2 -= 2)
832 *(volatile uint16_t *)(addr2) =
833 *(volatile uint16_t *)(addr1);
834 }
835 }
836}
837
838void
839bus_space_copy_region_4(bus_space_tag_t t, bus_space_handle_t h1,
840 bus_size_t o1, bus_space_handle_t h2,
841 bus_size_t o2, size_t c)
842{
843 vaddr_t addr1 = h1 + o1;
844 vaddr_t addr2 = h2 + o2;
845
846 BUS_SPACE_ADDRESS_SANITY(addr1, uint32_t, "bus addr 1");
847 BUS_SPACE_ADDRESS_SANITY(addr2, uint32_t, "bus addr 2");
848
849 if (x86_bus_space_is_io(t)) {
850 if (addr1 >= addr2) {
851 /* src after dest: copy forward */
852 for (; c != 0; c--, addr1 += 4, addr2 += 4)
853 outl(addr2, inl(addr1));
854 } else {
855 /* dest after src: copy backwards */
856 for (addr1 += 4 * (c - 1), addr2 += 4 * (c - 1);
857 c != 0; c--, addr1 -= 4, addr2 -= 4)
858 outl(addr2, inl(addr1));
859 }
860 } else {
861 if (addr1 >= addr2) {
862 /* src after dest: copy forward */
863 for (; c != 0; c--, addr1 += 4, addr2 += 4)
864 *(volatile uint32_t *)(addr2) =
865 *(volatile uint32_t *)(addr1);
866 } else {
867 /* dest after src: copy backwards */
868 for (addr1 += 4 * (c - 1), addr2 += 4 * (c - 1);
869 c != 0; c--, addr1 -= 4, addr2 -= 4)
870 *(volatile uint32_t *)(addr2) =
871 *(volatile uint32_t *)(addr1);
872 }
873 }
874}
875
876void
877bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t bsh,
878 bus_size_t offset, bus_size_t len, int flags)
879{
880
881 /* Function call is enough to prevent reordering of loads. */
882}
883
884void *
885bus_space_vaddr(bus_space_tag_t tag, bus_space_handle_t bsh)
886{
887
888 return x86_bus_space_is_mem(tag) ? (void *)bsh : NULL;
889}
890
891static const void *
892bit_to_function_pointer(const struct bus_space_overrides *ov, uint64_t bit)
893{
894 switch (bit) {
895 case BUS_SPACE_OVERRIDE_MAP:
896 return ov->ov_space_map;
897 case BUS_SPACE_OVERRIDE_UNMAP:
898 return ov->ov_space_unmap;
899 case BUS_SPACE_OVERRIDE_ALLOC:
900 return ov->ov_space_alloc;
901 case BUS_SPACE_OVERRIDE_FREE:
902 return ov->ov_space_free;
903 case BUS_SPACE_OVERRIDE_RESERVE:
904 return ov->ov_space_reserve;
905 case BUS_SPACE_OVERRIDE_RELEASE:
906 return ov->ov_space_release;
907 case BUS_SPACE_OVERRIDE_RESERVATION_MAP:
908 return ov->ov_space_reservation_map;
909 case BUS_SPACE_OVERRIDE_RESERVATION_UNMAP:
910 return ov->ov_space_reservation_unmap;
911 case BUS_SPACE_OVERRIDE_RESERVE_SUBREGION:
912 return ov->ov_space_reserve_subregion;
913 default:
914 return NULL;
915 }
916}
917
918void
919bus_space_tag_destroy(bus_space_tag_t bst)
920{
921 kmem_free(bst, sizeof(struct bus_space_tag));
922}
923
924int
925bus_space_tag_create(bus_space_tag_t obst, const uint64_t present,
926 const uint64_t extpresent, const struct bus_space_overrides *ov, void *ctx,
927 bus_space_tag_t *bstp)
928{
929 uint64_t bit, bits, nbits;
930 bus_space_tag_t bst;
931 const void *fp;
932
933 if (ov == NULL || present == 0 || extpresent != 0)
934 return EINVAL;
935
936 bst = kmem_alloc(sizeof(struct bus_space_tag), KM_SLEEP);
937
938 if (bst == NULL)
939 return ENOMEM;
940
941 bst->bst_super = obst;
942 bst->bst_type = obst->bst_type;
943
944 for (bits = present; bits != 0; bits = nbits) {
945 nbits = bits & (bits - 1);
946 bit = nbits ^ bits;
947 if ((fp = bit_to_function_pointer(ov, bit)) == NULL) {
948 printf("%s: missing bit %" PRIx64 "\n", __func__, bit);
949 goto einval;
950 }
951 }
952
953 bst->bst_ov = ov;
954 bst->bst_exists = obst->bst_exists | present;
955 bst->bst_present = present;
956 bst->bst_ctx = ctx;
957
958 *bstp = bst;
959
960 return 0;
961einval:
962 kmem_free(bst, sizeof(struct bus_space_tag));
963 return EINVAL;
964}
965