1/* $NetBSD: subr_lockdebug.c,v 1.54 2015/09/29 01:44:57 ozaki-r Exp $ */
2
3/*-
4 * Copyright (c) 2006, 2007, 2008 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Andrew Doran.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32/*
33 * Basic lock debugging code shared among lock primitives.
34 */
35
36#include <sys/cdefs.h>
37__KERNEL_RCSID(0, "$NetBSD: subr_lockdebug.c,v 1.54 2015/09/29 01:44:57 ozaki-r Exp $");
38
39#ifdef _KERNEL_OPT
40#include "opt_ddb.h"
41#endif
42
43#include <sys/param.h>
44#include <sys/proc.h>
45#include <sys/systm.h>
46#include <sys/kernel.h>
47#include <sys/kmem.h>
48#include <sys/lockdebug.h>
49#include <sys/sleepq.h>
50#include <sys/cpu.h>
51#include <sys/atomic.h>
52#include <sys/lock.h>
53#include <sys/rbtree.h>
54
55#include <machine/lock.h>
56
57unsigned int ld_panic;
58
59#ifdef LOCKDEBUG
60
61#define LD_BATCH_SHIFT 9
62#define LD_BATCH (1 << LD_BATCH_SHIFT)
63#define LD_BATCH_MASK (LD_BATCH - 1)
64#define LD_MAX_LOCKS 1048576
65#define LD_SLOP 16
66
67#define LD_LOCKED 0x01
68#define LD_SLEEPER 0x02
69
70#define LD_WRITE_LOCK 0x80000000
71
72typedef struct lockdebug {
73 struct rb_node ld_rb_node;
74 __cpu_simple_lock_t ld_spinlock;
75 _TAILQ_ENTRY(struct lockdebug, volatile) ld_chain;
76 _TAILQ_ENTRY(struct lockdebug, volatile) ld_achain;
77 volatile void *ld_lock;
78 lockops_t *ld_lockops;
79 struct lwp *ld_lwp;
80 uintptr_t ld_locked;
81 uintptr_t ld_unlocked;
82 uintptr_t ld_initaddr;
83 uint16_t ld_shares;
84 uint16_t ld_cpu;
85 uint8_t ld_flags;
86 uint8_t ld_shwant; /* advisory */
87 uint8_t ld_exwant; /* advisory */
88 uint8_t ld_unused;
89} volatile lockdebug_t;
90
91typedef _TAILQ_HEAD(lockdebuglist, struct lockdebug, volatile) lockdebuglist_t;
92
93__cpu_simple_lock_t ld_mod_lk;
94lockdebuglist_t ld_free = TAILQ_HEAD_INITIALIZER(ld_free);
95lockdebuglist_t ld_all = TAILQ_HEAD_INITIALIZER(ld_all);
96int ld_nfree;
97int ld_freeptr;
98int ld_recurse;
99bool ld_nomore;
100lockdebug_t ld_prime[LD_BATCH];
101
102static void lockdebug_abort1(lockdebug_t *, int, const char *,
103 const char *, bool);
104static int lockdebug_more(int);
105static void lockdebug_init(void);
106static void lockdebug_dump(lockdebug_t *, void (*)(const char *, ...)
107 __printflike(1, 2));
108
109static signed int
110ld_rbto_compare_nodes(void *ctx, const void *n1, const void *n2)
111{
112 const lockdebug_t *ld1 = n1;
113 const lockdebug_t *ld2 = n2;
114 const uintptr_t a = (uintptr_t)ld1->ld_lock;
115 const uintptr_t b = (uintptr_t)ld2->ld_lock;
116
117 if (a < b)
118 return -1;
119 if (a > b)
120 return 1;
121 return 0;
122}
123
124static signed int
125ld_rbto_compare_key(void *ctx, const void *n, const void *key)
126{
127 const lockdebug_t *ld = n;
128 const uintptr_t a = (uintptr_t)ld->ld_lock;
129 const uintptr_t b = (uintptr_t)key;
130
131 if (a < b)
132 return -1;
133 if (a > b)
134 return 1;
135 return 0;
136}
137
138static rb_tree_t ld_rb_tree;
139
140static const rb_tree_ops_t ld_rb_tree_ops = {
141 .rbto_compare_nodes = ld_rbto_compare_nodes,
142 .rbto_compare_key = ld_rbto_compare_key,
143 .rbto_node_offset = offsetof(lockdebug_t, ld_rb_node),
144 .rbto_context = NULL
145};
146
147static inline lockdebug_t *
148lockdebug_lookup1(volatile void *lock)
149{
150 lockdebug_t *ld;
151 struct cpu_info *ci;
152
153 ci = curcpu();
154 __cpu_simple_lock(&ci->ci_data.cpu_ld_lock);
155 ld = (lockdebug_t *)rb_tree_find_node(&ld_rb_tree, __UNVOLATILE(lock));
156 __cpu_simple_unlock(&ci->ci_data.cpu_ld_lock);
157 if (ld == NULL) {
158 return NULL;
159 }
160 __cpu_simple_lock(&ld->ld_spinlock);
161
162 return ld;
163}
164
165static void
166lockdebug_lock_cpus(void)
167{
168 CPU_INFO_ITERATOR cii;
169 struct cpu_info *ci;
170
171 for (CPU_INFO_FOREACH(cii, ci)) {
172 __cpu_simple_lock(&ci->ci_data.cpu_ld_lock);
173 }
174}
175
176static void
177lockdebug_unlock_cpus(void)
178{
179 CPU_INFO_ITERATOR cii;
180 struct cpu_info *ci;
181
182 for (CPU_INFO_FOREACH(cii, ci)) {
183 __cpu_simple_unlock(&ci->ci_data.cpu_ld_lock);
184 }
185}
186
187/*
188 * lockdebug_lookup:
189 *
190 * Find a lockdebug structure by a pointer to a lock and return it locked.
191 */
192static inline lockdebug_t *
193lockdebug_lookup(volatile void *lock, uintptr_t where)
194{
195 lockdebug_t *ld;
196
197 ld = lockdebug_lookup1(lock);
198 if (ld == NULL) {
199 panic("lockdebug_lookup: uninitialized lock "
200 "(lock=%p, from=%08"PRIxPTR")", lock, where);
201 }
202 return ld;
203}
204
205/*
206 * lockdebug_init:
207 *
208 * Initialize the lockdebug system. Allocate an initial pool of
209 * lockdebug structures before the VM system is up and running.
210 */
211static void
212lockdebug_init(void)
213{
214 lockdebug_t *ld;
215 int i;
216
217 TAILQ_INIT(&curcpu()->ci_data.cpu_ld_locks);
218 TAILQ_INIT(&curlwp->l_ld_locks);
219 __cpu_simple_lock_init(&curcpu()->ci_data.cpu_ld_lock);
220 __cpu_simple_lock_init(&ld_mod_lk);
221
222 rb_tree_init(&ld_rb_tree, &ld_rb_tree_ops);
223
224 ld = ld_prime;
225 for (i = 1, ld++; i < LD_BATCH; i++, ld++) {
226 __cpu_simple_lock_init(&ld->ld_spinlock);
227 TAILQ_INSERT_TAIL(&ld_free, ld, ld_chain);
228 TAILQ_INSERT_TAIL(&ld_all, ld, ld_achain);
229 }
230 ld_freeptr = 1;
231 ld_nfree = LD_BATCH - 1;
232}
233
234/*
235 * lockdebug_alloc:
236 *
237 * A lock is being initialized, so allocate an associated debug
238 * structure.
239 */
240bool
241lockdebug_alloc(volatile void *lock, lockops_t *lo, uintptr_t initaddr)
242{
243 struct cpu_info *ci;
244 lockdebug_t *ld;
245 int s;
246
247 if (lo == NULL || panicstr != NULL || ld_panic)
248 return false;
249 if (ld_freeptr == 0)
250 lockdebug_init();
251
252 s = splhigh();
253 __cpu_simple_lock(&ld_mod_lk);
254 if ((ld = lockdebug_lookup1(lock)) != NULL) {
255 __cpu_simple_unlock(&ld_mod_lk);
256 lockdebug_abort1(ld, s, __func__, "already initialized", true);
257 return false;
258 }
259
260 /*
261 * Pinch a new debug structure. We may recurse because we call
262 * kmem_alloc(), which may need to initialize new locks somewhere
263 * down the path. If not recursing, we try to maintain at least
264 * LD_SLOP structures free, which should hopefully be enough to
265 * satisfy kmem_alloc(). If we can't provide a structure, not to
266 * worry: we'll just mark the lock as not having an ID.
267 */
268 ci = curcpu();
269 ci->ci_lkdebug_recurse++;
270 if (TAILQ_EMPTY(&ld_free)) {
271 if (ci->ci_lkdebug_recurse > 1 || ld_nomore) {
272 ci->ci_lkdebug_recurse--;
273 __cpu_simple_unlock(&ld_mod_lk);
274 splx(s);
275 return false;
276 }
277 s = lockdebug_more(s);
278 } else if (ci->ci_lkdebug_recurse == 1 && ld_nfree < LD_SLOP) {
279 s = lockdebug_more(s);
280 }
281 if ((ld = TAILQ_FIRST(&ld_free)) == NULL) {
282 __cpu_simple_unlock(&ld_mod_lk);
283 splx(s);
284 return false;
285 }
286 TAILQ_REMOVE(&ld_free, ld, ld_chain);
287 ld_nfree--;
288 ci->ci_lkdebug_recurse--;
289
290 if (ld->ld_lock != NULL) {
291 panic("lockdebug_alloc: corrupt table ld %p", ld);
292 }
293
294 /* Initialise the structure. */
295 ld->ld_lock = lock;
296 ld->ld_lockops = lo;
297 ld->ld_locked = 0;
298 ld->ld_unlocked = 0;
299 ld->ld_lwp = NULL;
300 ld->ld_initaddr = initaddr;
301 ld->ld_flags = (lo->lo_type == LOCKOPS_SLEEP ? LD_SLEEPER : 0);
302 lockdebug_lock_cpus();
303 (void)rb_tree_insert_node(&ld_rb_tree, __UNVOLATILE(ld));
304 lockdebug_unlock_cpus();
305 __cpu_simple_unlock(&ld_mod_lk);
306
307 splx(s);
308 return true;
309}
310
311/*
312 * lockdebug_free:
313 *
314 * A lock is being destroyed, so release debugging resources.
315 */
316void
317lockdebug_free(volatile void *lock)
318{
319 lockdebug_t *ld;
320 int s;
321
322 if (panicstr != NULL || ld_panic)
323 return;
324
325 s = splhigh();
326 __cpu_simple_lock(&ld_mod_lk);
327 ld = lockdebug_lookup(lock, (uintptr_t) __builtin_return_address(0));
328 if (ld == NULL) {
329 __cpu_simple_unlock(&ld_mod_lk);
330 panic("lockdebug_free: destroying uninitialized object %p"
331 "(ld_lock=%p)", lock, ld->ld_lock);
332 return;
333 }
334 if ((ld->ld_flags & LD_LOCKED) != 0 || ld->ld_shares != 0) {
335 __cpu_simple_unlock(&ld_mod_lk);
336 lockdebug_abort1(ld, s, __func__, "is locked or in use", true);
337 return;
338 }
339 lockdebug_lock_cpus();
340 rb_tree_remove_node(&ld_rb_tree, __UNVOLATILE(ld));
341 lockdebug_unlock_cpus();
342 ld->ld_lock = NULL;
343 TAILQ_INSERT_TAIL(&ld_free, ld, ld_chain);
344 ld_nfree++;
345 __cpu_simple_unlock(&ld->ld_spinlock);
346 __cpu_simple_unlock(&ld_mod_lk);
347 splx(s);
348}
349
350/*
351 * lockdebug_more:
352 *
353 * Allocate a batch of debug structures and add to the free list.
354 * Must be called with ld_mod_lk held.
355 */
356static int
357lockdebug_more(int s)
358{
359 lockdebug_t *ld;
360 void *block;
361 int i, base, m;
362
363 /*
364 * Can't call kmem_alloc() if in interrupt context. XXX We could
365 * deadlock, because we don't know which locks the caller holds.
366 */
367 if (cpu_intr_p() || (curlwp->l_pflag & LP_INTR) != 0) {
368 return s;
369 }
370
371 while (ld_nfree < LD_SLOP) {
372 __cpu_simple_unlock(&ld_mod_lk);
373 splx(s);
374 block = kmem_zalloc(LD_BATCH * sizeof(lockdebug_t), KM_SLEEP);
375 s = splhigh();
376 __cpu_simple_lock(&ld_mod_lk);
377
378 if (block == NULL)
379 return s;
380
381 if (ld_nfree > LD_SLOP) {
382 /* Somebody beat us to it. */
383 __cpu_simple_unlock(&ld_mod_lk);
384 splx(s);
385 kmem_free(block, LD_BATCH * sizeof(lockdebug_t));
386 s = splhigh();
387 __cpu_simple_lock(&ld_mod_lk);
388 continue;
389 }
390
391 base = ld_freeptr;
392 ld_nfree += LD_BATCH;
393 ld = block;
394 base <<= LD_BATCH_SHIFT;
395 m = min(LD_MAX_LOCKS, base + LD_BATCH);
396
397 if (m == LD_MAX_LOCKS)
398 ld_nomore = true;
399
400 for (i = base; i < m; i++, ld++) {
401 __cpu_simple_lock_init(&ld->ld_spinlock);
402 TAILQ_INSERT_TAIL(&ld_free, ld, ld_chain);
403 TAILQ_INSERT_TAIL(&ld_all, ld, ld_achain);
404 }
405
406 membar_producer();
407 }
408
409 return s;
410}
411
412/*
413 * lockdebug_wantlock:
414 *
415 * Process the preamble to a lock acquire.
416 */
417void
418lockdebug_wantlock(volatile void *lock, uintptr_t where, int shared)
419{
420 struct lwp *l = curlwp;
421 lockdebug_t *ld;
422 bool recurse;
423 int s;
424
425 (void)shared;
426 recurse = false;
427
428 if (panicstr != NULL || ld_panic)
429 return;
430
431 s = splhigh();
432 if ((ld = lockdebug_lookup(lock, where)) == NULL) {
433 splx(s);
434 return;
435 }
436 if ((ld->ld_flags & LD_LOCKED) != 0 || ld->ld_shares != 0) {
437 if ((ld->ld_flags & LD_SLEEPER) != 0) {
438 if (ld->ld_lwp == l)
439 recurse = true;
440 } else if (ld->ld_cpu == (uint16_t)cpu_index(curcpu()))
441 recurse = true;
442 }
443 if (cpu_intr_p()) {
444 if ((ld->ld_flags & LD_SLEEPER) != 0) {
445 lockdebug_abort1(ld, s, __func__,
446 "acquiring sleep lock from interrupt context",
447 true);
448 return;
449 }
450 }
451 if (shared)
452 ld->ld_shwant++;
453 else
454 ld->ld_exwant++;
455 if (recurse) {
456 lockdebug_abort1(ld, s, __func__, "locking against myself",
457 true);
458 return;
459 }
460 __cpu_simple_unlock(&ld->ld_spinlock);
461 splx(s);
462}
463
464/*
465 * lockdebug_locked:
466 *
467 * Process a lock acquire operation.
468 */
469void
470lockdebug_locked(volatile void *lock, void *cvlock, uintptr_t where,
471 int shared)
472{
473 struct lwp *l = curlwp;
474 lockdebug_t *ld;
475 int s;
476
477 if (panicstr != NULL || ld_panic)
478 return;
479
480 s = splhigh();
481 if ((ld = lockdebug_lookup(lock, where)) == NULL) {
482 splx(s);
483 return;
484 }
485 if (cvlock) {
486 KASSERT(ld->ld_lockops->lo_type == LOCKOPS_CV);
487 if (lock == (void *)&lbolt) {
488 /* nothing */
489 } else if (ld->ld_shares++ == 0) {
490 ld->ld_locked = (uintptr_t)cvlock;
491 } else if (cvlock != (void *)ld->ld_locked) {
492 lockdebug_abort1(ld, s, __func__, "multiple locks used"
493 " with condition variable", true);
494 return;
495 }
496 } else if (shared) {
497 l->l_shlocks++;
498 ld->ld_locked = where;
499 ld->ld_shares++;
500 ld->ld_shwant--;
501 } else {
502 if ((ld->ld_flags & LD_LOCKED) != 0) {
503 lockdebug_abort1(ld, s, __func__, "already locked",
504 true);
505 return;
506 }
507 ld->ld_flags |= LD_LOCKED;
508 ld->ld_locked = where;
509 ld->ld_exwant--;
510 if ((ld->ld_flags & LD_SLEEPER) != 0) {
511 TAILQ_INSERT_TAIL(&l->l_ld_locks, ld, ld_chain);
512 } else {
513 TAILQ_INSERT_TAIL(&curcpu()->ci_data.cpu_ld_locks,
514 ld, ld_chain);
515 }
516 }
517 ld->ld_cpu = (uint16_t)cpu_index(curcpu());
518 ld->ld_lwp = l;
519 __cpu_simple_unlock(&ld->ld_spinlock);
520 splx(s);
521}
522
523/*
524 * lockdebug_unlocked:
525 *
526 * Process a lock release operation.
527 */
528void
529lockdebug_unlocked(volatile void *lock, uintptr_t where, int shared)
530{
531 struct lwp *l = curlwp;
532 lockdebug_t *ld;
533 int s;
534
535 if (panicstr != NULL || ld_panic)
536 return;
537
538 s = splhigh();
539 if ((ld = lockdebug_lookup(lock, where)) == NULL) {
540 splx(s);
541 return;
542 }
543 if (ld->ld_lockops->lo_type == LOCKOPS_CV) {
544 if (lock == (void *)&lbolt) {
545 /* nothing */
546 } else {
547 ld->ld_shares--;
548 }
549 } else if (shared) {
550 if (l->l_shlocks == 0) {
551 lockdebug_abort1(ld, s, __func__,
552 "no shared locks held by LWP", true);
553 return;
554 }
555 if (ld->ld_shares == 0) {
556 lockdebug_abort1(ld, s, __func__,
557 "no shared holds on this lock", true);
558 return;
559 }
560 l->l_shlocks--;
561 ld->ld_shares--;
562 if (ld->ld_lwp == l) {
563 ld->ld_unlocked = where;
564 ld->ld_lwp = NULL;
565 }
566 if (ld->ld_cpu == (uint16_t)cpu_index(curcpu()))
567 ld->ld_cpu = (uint16_t)-1;
568 } else {
569 if ((ld->ld_flags & LD_LOCKED) == 0) {
570 lockdebug_abort1(ld, s, __func__, "not locked", true);
571 return;
572 }
573
574 if ((ld->ld_flags & LD_SLEEPER) != 0) {
575 if (ld->ld_lwp != curlwp) {
576 lockdebug_abort1(ld, s, __func__,
577 "not held by current LWP", true);
578 return;
579 }
580 TAILQ_REMOVE(&l->l_ld_locks, ld, ld_chain);
581 } else {
582 if (ld->ld_cpu != (uint16_t)cpu_index(curcpu())) {
583 lockdebug_abort1(ld, s, __func__,
584 "not held by current CPU", true);
585 return;
586 }
587 TAILQ_REMOVE(&curcpu()->ci_data.cpu_ld_locks, ld,
588 ld_chain);
589 }
590 ld->ld_flags &= ~LD_LOCKED;
591 ld->ld_unlocked = where;
592 ld->ld_lwp = NULL;
593 }
594 __cpu_simple_unlock(&ld->ld_spinlock);
595 splx(s);
596}
597
598/*
599 * lockdebug_wakeup:
600 *
601 * Process a wakeup on a condition variable.
602 */
603void
604lockdebug_wakeup(volatile void *lock, uintptr_t where)
605{
606 lockdebug_t *ld;
607 int s;
608
609 if (panicstr != NULL || ld_panic || lock == (void *)&lbolt)
610 return;
611
612 s = splhigh();
613 /* Find the CV... */
614 if ((ld = lockdebug_lookup(lock, where)) == NULL) {
615 splx(s);
616 return;
617 }
618 /*
619 * If it has any waiters, ensure that they are using the
620 * same interlock.
621 */
622 if (ld->ld_shares != 0 && !mutex_owned((kmutex_t *)ld->ld_locked)) {
623 lockdebug_abort1(ld, s, __func__, "interlocking mutex not "
624 "held during wakeup", true);
625 return;
626 }
627 __cpu_simple_unlock(&ld->ld_spinlock);
628 splx(s);
629}
630
631/*
632 * lockdebug_barrier:
633 *
634 * Panic if we hold more than one specified spin lock, and optionally,
635 * if we hold sleep locks.
636 */
637void
638lockdebug_barrier(volatile void *spinlock, int slplocks)
639{
640 struct lwp *l = curlwp;
641 lockdebug_t *ld;
642 int s;
643
644 if (panicstr != NULL || ld_panic)
645 return;
646
647 s = splhigh();
648 if ((l->l_pflag & LP_INTR) == 0) {
649 TAILQ_FOREACH(ld, &curcpu()->ci_data.cpu_ld_locks, ld_chain) {
650 if (ld->ld_lock == spinlock) {
651 continue;
652 }
653 __cpu_simple_lock(&ld->ld_spinlock);
654 lockdebug_abort1(ld, s, __func__,
655 "spin lock held", true);
656 return;
657 }
658 }
659 if (slplocks) {
660 splx(s);
661 return;
662 }
663 if ((ld = TAILQ_FIRST(&l->l_ld_locks)) != NULL) {
664 __cpu_simple_lock(&ld->ld_spinlock);
665 lockdebug_abort1(ld, s, __func__, "sleep lock held", true);
666 return;
667 }
668 splx(s);
669 if (l->l_shlocks != 0) {
670 TAILQ_FOREACH(ld, &ld_all, ld_achain) {
671 if (ld->ld_lockops->lo_type == LOCKOPS_CV)
672 continue;
673 if (ld->ld_lwp == l)
674 lockdebug_dump(ld, printf);
675 }
676 panic("%s: holding %d shared locks", __func__, l->l_shlocks);
677 }
678}
679
680/*
681 * lockdebug_mem_check:
682 *
683 * Check for in-use locks within a memory region that is
684 * being freed.
685 */
686void
687lockdebug_mem_check(const char *func, void *base, size_t sz)
688{
689 lockdebug_t *ld;
690 struct cpu_info *ci;
691 int s;
692
693 if (panicstr != NULL || ld_panic)
694 return;
695
696 s = splhigh();
697 ci = curcpu();
698 __cpu_simple_lock(&ci->ci_data.cpu_ld_lock);
699 ld = (lockdebug_t *)rb_tree_find_node_geq(&ld_rb_tree, base);
700 if (ld != NULL) {
701 const uintptr_t lock = (uintptr_t)ld->ld_lock;
702
703 if ((uintptr_t)base > lock)
704 panic("%s: corrupt tree ld=%p, base=%p, sz=%zu",
705 __func__, ld, base, sz);
706 if (lock >= (uintptr_t)base + sz)
707 ld = NULL;
708 }
709 __cpu_simple_unlock(&ci->ci_data.cpu_ld_lock);
710 if (ld != NULL) {
711 __cpu_simple_lock(&ld->ld_spinlock);
712 lockdebug_abort1(ld, s, func,
713 "allocation contains active lock", !cold);
714 return;
715 }
716 splx(s);
717}
718
719/*
720 * lockdebug_dump:
721 *
722 * Dump information about a lock on panic, or for DDB.
723 */
724static void
725lockdebug_dump(lockdebug_t *ld, void (*pr)(const char *, ...)
726 __printflike(1, 2))
727{
728 int sleeper = (ld->ld_flags & LD_SLEEPER);
729
730 (*pr)(
731 "lock address : %#018lx type : %18s\n"
732 "initialized : %#018lx",
733 (long)ld->ld_lock, (sleeper ? "sleep/adaptive" : "spin"),
734 (long)ld->ld_initaddr);
735
736 if (ld->ld_lockops->lo_type == LOCKOPS_CV) {
737 (*pr)(" interlock: %#018lx\n", (long)ld->ld_locked);
738 } else {
739 (*pr)("\n"
740 "shared holds : %18u exclusive: %18u\n"
741 "shares wanted: %18u exclusive: %18u\n"
742 "current cpu : %18u last held: %18u\n"
743 "current lwp : %#018lx last held: %#018lx\n"
744 "last locked%c : %#018lx unlocked%c: %#018lx\n",
745 (unsigned)ld->ld_shares, ((ld->ld_flags & LD_LOCKED) != 0),
746 (unsigned)ld->ld_shwant, (unsigned)ld->ld_exwant,
747 (unsigned)cpu_index(curcpu()), (unsigned)ld->ld_cpu,
748 (long)curlwp, (long)ld->ld_lwp,
749 ((ld->ld_flags & LD_LOCKED) ? '*' : ' '),
750 (long)ld->ld_locked,
751 ((ld->ld_flags & LD_LOCKED) ? ' ' : '*'),
752 (long)ld->ld_unlocked);
753 }
754
755 if (ld->ld_lockops->lo_dump != NULL)
756 (*ld->ld_lockops->lo_dump)(ld->ld_lock);
757
758 if (sleeper) {
759 (*pr)("\n");
760 turnstile_print(ld->ld_lock, pr);
761 }
762}
763
764/*
765 * lockdebug_abort1:
766 *
767 * An error has been trapped - dump lock info and panic.
768 */
769static void
770lockdebug_abort1(lockdebug_t *ld, int s, const char *func,
771 const char *msg, bool dopanic)
772{
773
774 /*
775 * Don't make the situation worse if the system is already going
776 * down in flames. Once a panic is triggered, lockdebug state
777 * becomes stale and cannot be trusted.
778 */
779 if (atomic_inc_uint_nv(&ld_panic) != 1) {
780 __cpu_simple_unlock(&ld->ld_spinlock);
781 splx(s);
782 return;
783 }
784
785 printf_nolog("%s error: %s: %s\n\n", ld->ld_lockops->lo_name,
786 func, msg);
787 lockdebug_dump(ld, printf_nolog);
788 __cpu_simple_unlock(&ld->ld_spinlock);
789 splx(s);
790 printf_nolog("\n");
791 if (dopanic)
792 panic("LOCKDEBUG: %s error: %s: %s", ld->ld_lockops->lo_name,
793 func, msg);
794}
795
796#endif /* LOCKDEBUG */
797
798/*
799 * lockdebug_lock_print:
800 *
801 * Handle the DDB 'show lock' command.
802 */
803#ifdef DDB
804void
805lockdebug_lock_print(void *addr, void (*pr)(const char *, ...))
806{
807#ifdef LOCKDEBUG
808 lockdebug_t *ld;
809
810 TAILQ_FOREACH(ld, &ld_all, ld_achain) {
811 if (ld->ld_lock == NULL)
812 continue;
813 if (addr == NULL || ld->ld_lock == addr) {
814 lockdebug_dump(ld, pr);
815 if (addr != NULL)
816 return;
817 }
818 }
819 if (addr != NULL) {
820 (*pr)("Sorry, no record of a lock with address %p found.\n",
821 addr);
822 }
823#else
824 (*pr)("Sorry, kernel not built with the LOCKDEBUG option.\n");
825#endif /* LOCKDEBUG */
826}
827#endif /* DDB */
828
829/*
830 * lockdebug_abort:
831 *
832 * An error has been trapped - dump lock info and call panic().
833 */
834void
835lockdebug_abort(volatile void *lock, lockops_t *ops, const char *func,
836 const char *msg)
837{
838#ifdef LOCKDEBUG
839 lockdebug_t *ld;
840 int s;
841
842 s = splhigh();
843 if ((ld = lockdebug_lookup(lock,
844 (uintptr_t) __builtin_return_address(0))) != NULL) {
845 lockdebug_abort1(ld, s, func, msg, true);
846 return;
847 }
848 splx(s);
849#endif /* LOCKDEBUG */
850
851 /*
852 * Complain first on the occurrance only. Otherwise proceeed to
853 * panic where we will `rendezvous' with other CPUs if the machine
854 * is going down in flames.
855 */
856 if (atomic_inc_uint_nv(&ld_panic) == 1) {
857 printf_nolog("%s error: %s: %s\n\n"
858 "lock address : %#018lx\n"
859 "current cpu : %18d\n"
860 "current lwp : %#018lx\n",
861 ops->lo_name, func, msg, (long)lock,
862 (int)cpu_index(curcpu()), (long)curlwp);
863 (*ops->lo_dump)(lock);
864 printf_nolog("\n");
865 }
866
867 panic("lock error: %s: %s: %s: lock %p cpu %d lwp %p",
868 ops->lo_name, func, msg, lock, cpu_index(curcpu()), curlwp);
869}
870