1/* $NetBSD: kern_mutex.c,v 1.63 2016/07/07 06:55:43 msaitoh Exp $ */
2
3/*-
4 * Copyright (c) 2002, 2006, 2007, 2008 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Jason R. Thorpe and Andrew Doran.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32/*
33 * Kernel mutex implementation, modeled after those found in Solaris,
34 * a description of which can be found in:
35 *
36 * Solaris Internals: Core Kernel Architecture, Jim Mauro and
37 * Richard McDougall.
38 */
39
40#define __MUTEX_PRIVATE
41
42#include <sys/cdefs.h>
43__KERNEL_RCSID(0, "$NetBSD: kern_mutex.c,v 1.63 2016/07/07 06:55:43 msaitoh Exp $");
44
45#include <sys/param.h>
46#include <sys/atomic.h>
47#include <sys/proc.h>
48#include <sys/mutex.h>
49#include <sys/sched.h>
50#include <sys/sleepq.h>
51#include <sys/systm.h>
52#include <sys/lockdebug.h>
53#include <sys/kernel.h>
54#include <sys/intr.h>
55#include <sys/lock.h>
56#include <sys/types.h>
57
58#include <dev/lockstat.h>
59
60#include <machine/lock.h>
61
62/*
63 * When not running a debug kernel, spin mutexes are not much
64 * more than an splraiseipl() and splx() pair.
65 */
66
67#if defined(DIAGNOSTIC) || defined(MULTIPROCESSOR) || defined(LOCKDEBUG)
68#define FULL
69#endif
70
71/*
72 * Debugging support.
73 */
74
75#define MUTEX_WANTLOCK(mtx) \
76 LOCKDEBUG_WANTLOCK(MUTEX_DEBUG_P(mtx), (mtx), \
77 (uintptr_t)__builtin_return_address(0), 0)
78#define MUTEX_LOCKED(mtx) \
79 LOCKDEBUG_LOCKED(MUTEX_DEBUG_P(mtx), (mtx), NULL, \
80 (uintptr_t)__builtin_return_address(0), 0)
81#define MUTEX_UNLOCKED(mtx) \
82 LOCKDEBUG_UNLOCKED(MUTEX_DEBUG_P(mtx), (mtx), \
83 (uintptr_t)__builtin_return_address(0), 0)
84#define MUTEX_ABORT(mtx, msg) \
85 mutex_abort(mtx, __func__, msg)
86
87#if defined(LOCKDEBUG)
88
89#define MUTEX_DASSERT(mtx, cond) \
90do { \
91 if (!(cond)) \
92 MUTEX_ABORT(mtx, "assertion failed: " #cond); \
93} while (/* CONSTCOND */ 0);
94
95#else /* LOCKDEBUG */
96
97#define MUTEX_DASSERT(mtx, cond) /* nothing */
98
99#endif /* LOCKDEBUG */
100
101#if defined(DIAGNOSTIC)
102
103#define MUTEX_ASSERT(mtx, cond) \
104do { \
105 if (!(cond)) \
106 MUTEX_ABORT(mtx, "assertion failed: " #cond); \
107} while (/* CONSTCOND */ 0)
108
109#else /* DIAGNOSTIC */
110
111#define MUTEX_ASSERT(mtx, cond) /* nothing */
112
113#endif /* DIAGNOSTIC */
114
115/*
116 * Some architectures can't use __cpu_simple_lock as is so allow a way
117 * for them to use an alternate definition.
118 */
119#ifndef MUTEX_SPINBIT_LOCK_INIT
120#define MUTEX_SPINBIT_LOCK_INIT(mtx) __cpu_simple_lock_init(&(mtx)->mtx_lock)
121#endif
122#ifndef MUTEX_SPINBIT_LOCKED_P
123#define MUTEX_SPINBIT_LOCKED_P(mtx) __SIMPLELOCK_LOCKED_P(&(mtx)->mtx_lock)
124#endif
125#ifndef MUTEX_SPINBIT_LOCK_TRY
126#define MUTEX_SPINBIT_LOCK_TRY(mtx) __cpu_simple_lock_try(&(mtx)->mtx_lock)
127#endif
128#ifndef MUTEX_SPINBIT_LOCK_UNLOCK
129#define MUTEX_SPINBIT_LOCK_UNLOCK(mtx) __cpu_simple_unlock(&(mtx)->mtx_lock)
130#endif
131
132#ifndef MUTEX_INITIALIZE_SPIN_IPL
133#define MUTEX_INITIALIZE_SPIN_IPL(mtx, ipl) \
134 ((mtx)->mtx_ipl = makeiplcookie((ipl)))
135#endif
136
137/*
138 * Spin mutex SPL save / restore.
139 */
140
141#define MUTEX_SPIN_SPLRAISE(mtx) \
142do { \
143 struct cpu_info *x__ci; \
144 int x__cnt, s; \
145 s = splraiseipl(MUTEX_SPIN_IPL(mtx)); \
146 x__ci = curcpu(); \
147 x__cnt = x__ci->ci_mtx_count--; \
148 __insn_barrier(); \
149 if (x__cnt == 0) \
150 x__ci->ci_mtx_oldspl = (s); \
151} while (/* CONSTCOND */ 0)
152
153#define MUTEX_SPIN_SPLRESTORE(mtx) \
154do { \
155 struct cpu_info *x__ci = curcpu(); \
156 int s = x__ci->ci_mtx_oldspl; \
157 __insn_barrier(); \
158 if (++(x__ci->ci_mtx_count) == 0) \
159 splx(s); \
160} while (/* CONSTCOND */ 0)
161
162/*
163 * For architectures that provide 'simple' mutexes: they provide a
164 * CAS function that is either MP-safe, or does not need to be MP
165 * safe. Adaptive mutexes on these architectures do not require an
166 * additional interlock.
167 */
168
169#ifdef __HAVE_SIMPLE_MUTEXES
170
171#define MUTEX_OWNER(owner) \
172 (owner & MUTEX_THREAD)
173#define MUTEX_HAS_WAITERS(mtx) \
174 (((int)(mtx)->mtx_owner & MUTEX_BIT_WAITERS) != 0)
175
176#define MUTEX_INITIALIZE_ADAPTIVE(mtx, dodebug) \
177 if (!dodebug) \
178 (mtx)->mtx_owner |= MUTEX_BIT_NODEBUG; \
179do { \
180} while (/* CONSTCOND */ 0);
181
182#define MUTEX_INITIALIZE_SPIN(mtx, dodebug, ipl) \
183do { \
184 (mtx)->mtx_owner = MUTEX_BIT_SPIN; \
185 if (!dodebug) \
186 (mtx)->mtx_owner |= MUTEX_BIT_NODEBUG; \
187 MUTEX_INITIALIZE_SPIN_IPL((mtx), (ipl)); \
188 MUTEX_SPINBIT_LOCK_INIT((mtx)); \
189} while (/* CONSTCOND */ 0)
190
191#define MUTEX_DESTROY(mtx) \
192do { \
193 (mtx)->mtx_owner = MUTEX_THREAD; \
194} while (/* CONSTCOND */ 0);
195
196#define MUTEX_SPIN_P(mtx) \
197 (((mtx)->mtx_owner & MUTEX_BIT_SPIN) != 0)
198#define MUTEX_ADAPTIVE_P(mtx) \
199 (((mtx)->mtx_owner & MUTEX_BIT_SPIN) == 0)
200
201#define MUTEX_DEBUG_P(mtx) (((mtx)->mtx_owner & MUTEX_BIT_NODEBUG) == 0)
202#if defined(LOCKDEBUG)
203#define MUTEX_OWNED(owner) (((owner) & ~MUTEX_BIT_NODEBUG) != 0)
204#define MUTEX_INHERITDEBUG(n, o) (n) |= (o) & MUTEX_BIT_NODEBUG
205#else /* defined(LOCKDEBUG) */
206#define MUTEX_OWNED(owner) ((owner) != 0)
207#define MUTEX_INHERITDEBUG(n, o) /* nothing */
208#endif /* defined(LOCKDEBUG) */
209
210static inline int
211MUTEX_ACQUIRE(kmutex_t *mtx, uintptr_t curthread)
212{
213 int rv;
214 uintptr_t oldown = 0;
215 uintptr_t newown = curthread;
216
217 MUTEX_INHERITDEBUG(oldown, mtx->mtx_owner);
218 MUTEX_INHERITDEBUG(newown, oldown);
219 rv = MUTEX_CAS(&mtx->mtx_owner, oldown, newown);
220 MUTEX_RECEIVE(mtx);
221 return rv;
222}
223
224static inline int
225MUTEX_SET_WAITERS(kmutex_t *mtx, uintptr_t owner)
226{
227 int rv;
228 rv = MUTEX_CAS(&mtx->mtx_owner, owner, owner | MUTEX_BIT_WAITERS);
229 MUTEX_RECEIVE(mtx);
230 return rv;
231}
232
233static inline void
234MUTEX_RELEASE(kmutex_t *mtx)
235{
236 uintptr_t newown;
237
238 MUTEX_GIVE(mtx);
239 newown = 0;
240 MUTEX_INHERITDEBUG(newown, mtx->mtx_owner);
241 mtx->mtx_owner = newown;
242}
243#endif /* __HAVE_SIMPLE_MUTEXES */
244
245/*
246 * Patch in stubs via strong alias where they are not available.
247 */
248
249#if defined(LOCKDEBUG)
250#undef __HAVE_MUTEX_STUBS
251#undef __HAVE_SPIN_MUTEX_STUBS
252#endif
253
254#ifndef __HAVE_MUTEX_STUBS
255__strong_alias(mutex_enter,mutex_vector_enter);
256__strong_alias(mutex_exit,mutex_vector_exit);
257#endif
258
259#ifndef __HAVE_SPIN_MUTEX_STUBS
260__strong_alias(mutex_spin_enter,mutex_vector_enter);
261__strong_alias(mutex_spin_exit,mutex_vector_exit);
262#endif
263
264static void mutex_abort(kmutex_t *, const char *, const char *);
265static void mutex_dump(volatile void *);
266
267lockops_t mutex_spin_lockops = {
268 "Mutex",
269 LOCKOPS_SPIN,
270 mutex_dump
271};
272
273lockops_t mutex_adaptive_lockops = {
274 "Mutex",
275 LOCKOPS_SLEEP,
276 mutex_dump
277};
278
279syncobj_t mutex_syncobj = {
280 SOBJ_SLEEPQ_SORTED,
281 turnstile_unsleep,
282 turnstile_changepri,
283 sleepq_lendpri,
284 (void *)mutex_owner,
285};
286
287/*
288 * mutex_dump:
289 *
290 * Dump the contents of a mutex structure.
291 */
292void
293mutex_dump(volatile void *cookie)
294{
295 volatile kmutex_t *mtx = cookie;
296
297 printf_nolog("owner field : %#018lx wait/spin: %16d/%d\n",
298 (long)MUTEX_OWNER(mtx->mtx_owner), MUTEX_HAS_WAITERS(mtx),
299 MUTEX_SPIN_P(mtx));
300}
301
302/*
303 * mutex_abort:
304 *
305 * Dump information about an error and panic the system. This
306 * generates a lot of machine code in the DIAGNOSTIC case, so
307 * we ask the compiler to not inline it.
308 */
309void __noinline
310mutex_abort(kmutex_t *mtx, const char *func, const char *msg)
311{
312
313 LOCKDEBUG_ABORT(mtx, (MUTEX_SPIN_P(mtx) ?
314 &mutex_spin_lockops : &mutex_adaptive_lockops), func, msg);
315}
316
317/*
318 * mutex_init:
319 *
320 * Initialize a mutex for use. Note that adaptive mutexes are in
321 * essence spin mutexes that can sleep to avoid deadlock and wasting
322 * CPU time. We can't easily provide a type of mutex that always
323 * sleeps - see comments in mutex_vector_enter() about releasing
324 * mutexes unlocked.
325 */
326void
327mutex_init(kmutex_t *mtx, kmutex_type_t type, int ipl)
328{
329 bool dodebug;
330
331 memset(mtx, 0, sizeof(*mtx));
332
333 switch (type) {
334 case MUTEX_ADAPTIVE:
335 KASSERT(ipl == IPL_NONE);
336 break;
337 case MUTEX_DEFAULT:
338 case MUTEX_DRIVER:
339 if (ipl == IPL_NONE || ipl == IPL_SOFTCLOCK ||
340 ipl == IPL_SOFTBIO || ipl == IPL_SOFTNET ||
341 ipl == IPL_SOFTSERIAL) {
342 type = MUTEX_ADAPTIVE;
343 } else {
344 type = MUTEX_SPIN;
345 }
346 break;
347 default:
348 break;
349 }
350
351 switch (type) {
352 case MUTEX_NODEBUG:
353 dodebug = LOCKDEBUG_ALLOC(mtx, NULL,
354 (uintptr_t)__builtin_return_address(0));
355 MUTEX_INITIALIZE_SPIN(mtx, dodebug, ipl);
356 break;
357 case MUTEX_ADAPTIVE:
358 dodebug = LOCKDEBUG_ALLOC(mtx, &mutex_adaptive_lockops,
359 (uintptr_t)__builtin_return_address(0));
360 MUTEX_INITIALIZE_ADAPTIVE(mtx, dodebug);
361 break;
362 case MUTEX_SPIN:
363 dodebug = LOCKDEBUG_ALLOC(mtx, &mutex_spin_lockops,
364 (uintptr_t)__builtin_return_address(0));
365 MUTEX_INITIALIZE_SPIN(mtx, dodebug, ipl);
366 break;
367 default:
368 panic("mutex_init: impossible type");
369 break;
370 }
371}
372
373/*
374 * mutex_destroy:
375 *
376 * Tear down a mutex.
377 */
378void
379mutex_destroy(kmutex_t *mtx)
380{
381
382 if (MUTEX_ADAPTIVE_P(mtx)) {
383 MUTEX_ASSERT(mtx, !MUTEX_OWNED(mtx->mtx_owner) &&
384 !MUTEX_HAS_WAITERS(mtx));
385 } else {
386 MUTEX_ASSERT(mtx, !MUTEX_SPINBIT_LOCKED_P(mtx));
387 }
388
389 LOCKDEBUG_FREE(MUTEX_DEBUG_P(mtx), mtx);
390 MUTEX_DESTROY(mtx);
391}
392
393#ifdef MULTIPROCESSOR
394/*
395 * mutex_oncpu:
396 *
397 * Return true if an adaptive mutex owner is running on a CPU in the
398 * system. If the target is waiting on the kernel big lock, then we
399 * must release it. This is necessary to avoid deadlock.
400 */
401static bool
402mutex_oncpu(uintptr_t owner)
403{
404 struct cpu_info *ci;
405 lwp_t *l;
406
407 KASSERT(kpreempt_disabled());
408
409 if (!MUTEX_OWNED(owner)) {
410 return false;
411 }
412
413 /*
414 * See lwp_dtor() why dereference of the LWP pointer is safe.
415 * We must have kernel preemption disabled for that.
416 */
417 l = (lwp_t *)MUTEX_OWNER(owner);
418 ci = l->l_cpu;
419
420 if (ci && ci->ci_curlwp == l) {
421 /* Target is running; do we need to block? */
422 return (ci->ci_biglock_wanted != l);
423 }
424
425 /* Not running. It may be safe to block now. */
426 return false;
427}
428#endif /* MULTIPROCESSOR */
429
430/*
431 * mutex_vector_enter:
432 *
433 * Support routine for mutex_enter() that must handle all cases. In
434 * the LOCKDEBUG case, mutex_enter() is always aliased here, even if
435 * fast-path stubs are available. If a mutex_spin_enter() stub is
436 * not available, then it is also aliased directly here.
437 */
438void
439mutex_vector_enter(kmutex_t *mtx)
440{
441 uintptr_t owner, curthread;
442 turnstile_t *ts;
443#ifdef MULTIPROCESSOR
444 u_int count;
445#endif
446 LOCKSTAT_COUNTER(spincnt);
447 LOCKSTAT_COUNTER(slpcnt);
448 LOCKSTAT_TIMER(spintime);
449 LOCKSTAT_TIMER(slptime);
450 LOCKSTAT_FLAG(lsflag);
451
452 /*
453 * Handle spin mutexes.
454 */
455 if (MUTEX_SPIN_P(mtx)) {
456#if defined(LOCKDEBUG) && defined(MULTIPROCESSOR)
457 u_int spins = 0;
458#endif
459 MUTEX_SPIN_SPLRAISE(mtx);
460 MUTEX_WANTLOCK(mtx);
461#ifdef FULL
462 if (MUTEX_SPINBIT_LOCK_TRY(mtx)) {
463 MUTEX_LOCKED(mtx);
464 return;
465 }
466#if !defined(MULTIPROCESSOR)
467 MUTEX_ABORT(mtx, "locking against myself");
468#else /* !MULTIPROCESSOR */
469
470 LOCKSTAT_ENTER(lsflag);
471 LOCKSTAT_START_TIMER(lsflag, spintime);
472 count = SPINLOCK_BACKOFF_MIN;
473
474 /*
475 * Spin testing the lock word and do exponential backoff
476 * to reduce cache line ping-ponging between CPUs.
477 */
478 do {
479 if (panicstr != NULL)
480 break;
481 while (MUTEX_SPINBIT_LOCKED_P(mtx)) {
482 SPINLOCK_BACKOFF(count);
483#ifdef LOCKDEBUG
484 if (SPINLOCK_SPINOUT(spins))
485 MUTEX_ABORT(mtx, "spinout");
486#endif /* LOCKDEBUG */
487 }
488 } while (!MUTEX_SPINBIT_LOCK_TRY(mtx));
489
490 if (count != SPINLOCK_BACKOFF_MIN) {
491 LOCKSTAT_STOP_TIMER(lsflag, spintime);
492 LOCKSTAT_EVENT(lsflag, mtx,
493 LB_SPIN_MUTEX | LB_SPIN, 1, spintime);
494 }
495 LOCKSTAT_EXIT(lsflag);
496#endif /* !MULTIPROCESSOR */
497#endif /* FULL */
498 MUTEX_LOCKED(mtx);
499 return;
500 }
501
502 curthread = (uintptr_t)curlwp;
503
504 MUTEX_DASSERT(mtx, MUTEX_ADAPTIVE_P(mtx));
505 MUTEX_ASSERT(mtx, curthread != 0);
506 MUTEX_WANTLOCK(mtx);
507
508 if (panicstr == NULL) {
509 LOCKDEBUG_BARRIER(&kernel_lock, 1);
510 }
511
512 LOCKSTAT_ENTER(lsflag);
513
514 /*
515 * Adaptive mutex; spin trying to acquire the mutex. If we
516 * determine that the owner is not running on a processor,
517 * then we stop spinning, and sleep instead.
518 */
519 KPREEMPT_DISABLE(curlwp);
520 for (owner = mtx->mtx_owner;;) {
521 if (!MUTEX_OWNED(owner)) {
522 /*
523 * Mutex owner clear could mean two things:
524 *
525 * * The mutex has been released.
526 * * The owner field hasn't been set yet.
527 *
528 * Try to acquire it again. If that fails,
529 * we'll just loop again.
530 */
531 if (MUTEX_ACQUIRE(mtx, curthread))
532 break;
533 owner = mtx->mtx_owner;
534 continue;
535 }
536 if (__predict_false(panicstr != NULL)) {
537 KPREEMPT_ENABLE(curlwp);
538 return;
539 }
540 if (__predict_false(MUTEX_OWNER(owner) == curthread)) {
541 MUTEX_ABORT(mtx, "locking against myself");
542 }
543#ifdef MULTIPROCESSOR
544 /*
545 * Check to see if the owner is running on a processor.
546 * If so, then we should just spin, as the owner will
547 * likely release the lock very soon.
548 */
549 if (mutex_oncpu(owner)) {
550 LOCKSTAT_START_TIMER(lsflag, spintime);
551 count = SPINLOCK_BACKOFF_MIN;
552 do {
553 KPREEMPT_ENABLE(curlwp);
554 SPINLOCK_BACKOFF(count);
555 KPREEMPT_DISABLE(curlwp);
556 owner = mtx->mtx_owner;
557 } while (mutex_oncpu(owner));
558 LOCKSTAT_STOP_TIMER(lsflag, spintime);
559 LOCKSTAT_COUNT(spincnt, 1);
560 if (!MUTEX_OWNED(owner))
561 continue;
562 }
563#endif
564
565 ts = turnstile_lookup(mtx);
566
567 /*
568 * Once we have the turnstile chain interlock, mark the
569 * mutex has having waiters. If that fails, spin again:
570 * chances are that the mutex has been released.
571 */
572 if (!MUTEX_SET_WAITERS(mtx, owner)) {
573 turnstile_exit(mtx);
574 owner = mtx->mtx_owner;
575 continue;
576 }
577
578#ifdef MULTIPROCESSOR
579 /*
580 * mutex_exit() is permitted to release the mutex without
581 * any interlocking instructions, and the following can
582 * occur as a result:
583 *
584 * CPU 1: MUTEX_SET_WAITERS() CPU2: mutex_exit()
585 * ---------------------------- ----------------------------
586 * .. acquire cache line
587 * .. test for waiters
588 * acquire cache line <- lose cache line
589 * lock cache line ..
590 * verify mutex is held ..
591 * set waiters ..
592 * unlock cache line ..
593 * lose cache line -> acquire cache line
594 * .. clear lock word, waiters
595 * return success
596 *
597 * There is another race that can occur: a third CPU could
598 * acquire the mutex as soon as it is released. Since
599 * adaptive mutexes are primarily spin mutexes, this is not
600 * something that we need to worry about too much. What we
601 * do need to ensure is that the waiters bit gets set.
602 *
603 * To allow the unlocked release, we need to make some
604 * assumptions here:
605 *
606 * o Release is the only non-atomic/unlocked operation
607 * that can be performed on the mutex. (It must still
608 * be atomic on the local CPU, e.g. in case interrupted
609 * or preempted).
610 *
611 * o At any given time, MUTEX_SET_WAITERS() can only ever
612 * be in progress on one CPU in the system - guaranteed
613 * by the turnstile chain lock.
614 *
615 * o No other operations other than MUTEX_SET_WAITERS()
616 * and release can modify a mutex with a non-zero
617 * owner field.
618 *
619 * o The result of a successful MUTEX_SET_WAITERS() call
620 * is an unbuffered write that is immediately visible
621 * to all other processors in the system.
622 *
623 * o If the holding LWP switches away, it posts a store
624 * fence before changing curlwp, ensuring that any
625 * overwrite of the mutex waiters flag by mutex_exit()
626 * completes before the modification of curlwp becomes
627 * visible to this CPU.
628 *
629 * o mi_switch() posts a store fence before setting curlwp
630 * and before resuming execution of an LWP.
631 *
632 * o _kernel_lock() posts a store fence before setting
633 * curcpu()->ci_biglock_wanted, and after clearing it.
634 * This ensures that any overwrite of the mutex waiters
635 * flag by mutex_exit() completes before the modification
636 * of ci_biglock_wanted becomes visible.
637 *
638 * We now post a read memory barrier (after setting the
639 * waiters field) and check the lock holder's status again.
640 * Some of the possible outcomes (not an exhaustive list):
641 *
642 * 1. The on-CPU check returns true: the holding LWP is
643 * running again. The lock may be released soon and
644 * we should spin. Importantly, we can't trust the
645 * value of the waiters flag.
646 *
647 * 2. The on-CPU check returns false: the holding LWP is
648 * not running. We now have the opportunity to check
649 * if mutex_exit() has blatted the modifications made
650 * by MUTEX_SET_WAITERS().
651 *
652 * 3. The on-CPU check returns false: the holding LWP may
653 * or may not be running. It has context switched at
654 * some point during our check. Again, we have the
655 * chance to see if the waiters bit is still set or
656 * has been overwritten.
657 *
658 * 4. The on-CPU check returns false: the holding LWP is
659 * running on a CPU, but wants the big lock. It's OK
660 * to check the waiters field in this case.
661 *
662 * 5. The has-waiters check fails: the mutex has been
663 * released, the waiters flag cleared and another LWP
664 * now owns the mutex.
665 *
666 * 6. The has-waiters check fails: the mutex has been
667 * released.
668 *
669 * If the waiters bit is not set it's unsafe to go asleep,
670 * as we might never be awoken.
671 */
672 if ((membar_consumer(), mutex_oncpu(owner)) ||
673 (membar_consumer(), !MUTEX_HAS_WAITERS(mtx))) {
674 turnstile_exit(mtx);
675 owner = mtx->mtx_owner;
676 continue;
677 }
678#endif /* MULTIPROCESSOR */
679
680 LOCKSTAT_START_TIMER(lsflag, slptime);
681
682 turnstile_block(ts, TS_WRITER_Q, mtx, &mutex_syncobj);
683
684 LOCKSTAT_STOP_TIMER(lsflag, slptime);
685 LOCKSTAT_COUNT(slpcnt, 1);
686
687 owner = mtx->mtx_owner;
688 }
689 KPREEMPT_ENABLE(curlwp);
690
691 LOCKSTAT_EVENT(lsflag, mtx, LB_ADAPTIVE_MUTEX | LB_SLEEP1,
692 slpcnt, slptime);
693 LOCKSTAT_EVENT(lsflag, mtx, LB_ADAPTIVE_MUTEX | LB_SPIN,
694 spincnt, spintime);
695 LOCKSTAT_EXIT(lsflag);
696
697 MUTEX_DASSERT(mtx, MUTEX_OWNER(mtx->mtx_owner) == curthread);
698 MUTEX_LOCKED(mtx);
699}
700
701/*
702 * mutex_vector_exit:
703 *
704 * Support routine for mutex_exit() that handles all cases.
705 */
706void
707mutex_vector_exit(kmutex_t *mtx)
708{
709 turnstile_t *ts;
710 uintptr_t curthread;
711
712 if (MUTEX_SPIN_P(mtx)) {
713#ifdef FULL
714 if (__predict_false(!MUTEX_SPINBIT_LOCKED_P(mtx))) {
715 if (panicstr != NULL)
716 return;
717 MUTEX_ABORT(mtx, "exiting unheld spin mutex");
718 }
719 MUTEX_UNLOCKED(mtx);
720 MUTEX_SPINBIT_LOCK_UNLOCK(mtx);
721#endif
722 MUTEX_SPIN_SPLRESTORE(mtx);
723 return;
724 }
725
726 if (__predict_false((uintptr_t)panicstr | cold)) {
727 MUTEX_UNLOCKED(mtx);
728 MUTEX_RELEASE(mtx);
729 return;
730 }
731
732 curthread = (uintptr_t)curlwp;
733 MUTEX_DASSERT(mtx, curthread != 0);
734 MUTEX_ASSERT(mtx, MUTEX_OWNER(mtx->mtx_owner) == curthread);
735 MUTEX_UNLOCKED(mtx);
736#if !defined(LOCKDEBUG)
737 __USE(curthread);
738#endif
739
740#ifdef LOCKDEBUG
741 /*
742 * Avoid having to take the turnstile chain lock every time
743 * around. Raise the priority level to splhigh() in order
744 * to disable preemption and so make the following atomic.
745 */
746 {
747 int s = splhigh();
748 if (!MUTEX_HAS_WAITERS(mtx)) {
749 MUTEX_RELEASE(mtx);
750 splx(s);
751 return;
752 }
753 splx(s);
754 }
755#endif
756
757 /*
758 * Get this lock's turnstile. This gets the interlock on
759 * the sleep queue. Once we have that, we can clear the
760 * lock. If there was no turnstile for the lock, there
761 * were no waiters remaining.
762 */
763 ts = turnstile_lookup(mtx);
764
765 if (ts == NULL) {
766 MUTEX_RELEASE(mtx);
767 turnstile_exit(mtx);
768 } else {
769 MUTEX_RELEASE(mtx);
770 turnstile_wakeup(ts, TS_WRITER_Q,
771 TS_WAITERS(ts, TS_WRITER_Q), NULL);
772 }
773}
774
775#ifndef __HAVE_SIMPLE_MUTEXES
776/*
777 * mutex_wakeup:
778 *
779 * Support routine for mutex_exit() that wakes up all waiters.
780 * We assume that the mutex has been released, but it need not
781 * be.
782 */
783void
784mutex_wakeup(kmutex_t *mtx)
785{
786 turnstile_t *ts;
787
788 ts = turnstile_lookup(mtx);
789 if (ts == NULL) {
790 turnstile_exit(mtx);
791 return;
792 }
793 MUTEX_CLEAR_WAITERS(mtx);
794 turnstile_wakeup(ts, TS_WRITER_Q, TS_WAITERS(ts, TS_WRITER_Q), NULL);
795}
796#endif /* !__HAVE_SIMPLE_MUTEXES */
797
798/*
799 * mutex_owned:
800 *
801 * Return true if the current LWP (adaptive) or CPU (spin)
802 * holds the mutex.
803 */
804int
805mutex_owned(kmutex_t *mtx)
806{
807
808 if (mtx == NULL)
809 return 0;
810 if (MUTEX_ADAPTIVE_P(mtx))
811 return MUTEX_OWNER(mtx->mtx_owner) == (uintptr_t)curlwp;
812#ifdef FULL
813 return MUTEX_SPINBIT_LOCKED_P(mtx);
814#else
815 return 1;
816#endif
817}
818
819/*
820 * mutex_owner:
821 *
822 * Return the current owner of an adaptive mutex. Used for
823 * priority inheritance.
824 */
825lwp_t *
826mutex_owner(kmutex_t *mtx)
827{
828
829 MUTEX_ASSERT(mtx, MUTEX_ADAPTIVE_P(mtx));
830 return (struct lwp *)MUTEX_OWNER(mtx->mtx_owner);
831}
832
833/*
834 * mutex_tryenter:
835 *
836 * Try to acquire the mutex; return non-zero if we did.
837 */
838int
839mutex_tryenter(kmutex_t *mtx)
840{
841 uintptr_t curthread;
842
843 /*
844 * Handle spin mutexes.
845 */
846 if (MUTEX_SPIN_P(mtx)) {
847 MUTEX_SPIN_SPLRAISE(mtx);
848#ifdef FULL
849 if (MUTEX_SPINBIT_LOCK_TRY(mtx)) {
850 MUTEX_WANTLOCK(mtx);
851 MUTEX_LOCKED(mtx);
852 return 1;
853 }
854 MUTEX_SPIN_SPLRESTORE(mtx);
855#else
856 MUTEX_WANTLOCK(mtx);
857 MUTEX_LOCKED(mtx);
858 return 1;
859#endif
860 } else {
861 curthread = (uintptr_t)curlwp;
862 MUTEX_ASSERT(mtx, curthread != 0);
863 if (MUTEX_ACQUIRE(mtx, curthread)) {
864 MUTEX_WANTLOCK(mtx);
865 MUTEX_LOCKED(mtx);
866 MUTEX_DASSERT(mtx,
867 MUTEX_OWNER(mtx->mtx_owner) == curthread);
868 return 1;
869 }
870 }
871
872 return 0;
873}
874
875#if defined(__HAVE_SPIN_MUTEX_STUBS) || defined(FULL)
876/*
877 * mutex_spin_retry:
878 *
879 * Support routine for mutex_spin_enter(). Assumes that the caller
880 * has already raised the SPL, and adjusted counters.
881 */
882void
883mutex_spin_retry(kmutex_t *mtx)
884{
885#ifdef MULTIPROCESSOR
886 u_int count;
887 LOCKSTAT_TIMER(spintime);
888 LOCKSTAT_FLAG(lsflag);
889#ifdef LOCKDEBUG
890 u_int spins = 0;
891#endif /* LOCKDEBUG */
892
893 MUTEX_WANTLOCK(mtx);
894
895 LOCKSTAT_ENTER(lsflag);
896 LOCKSTAT_START_TIMER(lsflag, spintime);
897 count = SPINLOCK_BACKOFF_MIN;
898
899 /*
900 * Spin testing the lock word and do exponential backoff
901 * to reduce cache line ping-ponging between CPUs.
902 */
903 do {
904 if (panicstr != NULL)
905 break;
906 while (MUTEX_SPINBIT_LOCKED_P(mtx)) {
907 SPINLOCK_BACKOFF(count);
908#ifdef LOCKDEBUG
909 if (SPINLOCK_SPINOUT(spins))
910 MUTEX_ABORT(mtx, "spinout");
911#endif /* LOCKDEBUG */
912 }
913 } while (!MUTEX_SPINBIT_LOCK_TRY(mtx));
914
915 LOCKSTAT_STOP_TIMER(lsflag, spintime);
916 LOCKSTAT_EVENT(lsflag, mtx, LB_SPIN_MUTEX | LB_SPIN, 1, spintime);
917 LOCKSTAT_EXIT(lsflag);
918
919 MUTEX_LOCKED(mtx);
920#else /* MULTIPROCESSOR */
921 MUTEX_ABORT(mtx, "locking against myself");
922#endif /* MULTIPROCESSOR */
923}
924#endif /* defined(__HAVE_SPIN_MUTEX_STUBS) || defined(FULL) */
925