1/* $NetBSD: kern_rwlock.c,v 1.45 2014/11/28 08:28:17 uebayasi Exp $ */
2
3/*-
4 * Copyright (c) 2002, 2006, 2007, 2008, 2009 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Jason R. Thorpe and Andrew Doran.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32/*
33 * Kernel reader/writer lock implementation, modeled after those
34 * found in Solaris, a description of which can be found in:
35 *
36 * Solaris Internals: Core Kernel Architecture, Jim Mauro and
37 * Richard McDougall.
38 */
39
40#include <sys/cdefs.h>
41__KERNEL_RCSID(0, "$NetBSD: kern_rwlock.c,v 1.45 2014/11/28 08:28:17 uebayasi Exp $");
42
43#define __RWLOCK_PRIVATE
44
45#include <sys/param.h>
46#include <sys/proc.h>
47#include <sys/rwlock.h>
48#include <sys/sched.h>
49#include <sys/sleepq.h>
50#include <sys/systm.h>
51#include <sys/lockdebug.h>
52#include <sys/cpu.h>
53#include <sys/atomic.h>
54#include <sys/lock.h>
55
56#include <dev/lockstat.h>
57
58/*
59 * LOCKDEBUG
60 */
61
62#if defined(LOCKDEBUG)
63
64#define RW_WANTLOCK(rw, op) \
65 LOCKDEBUG_WANTLOCK(RW_DEBUG_P(rw), (rw), \
66 (uintptr_t)__builtin_return_address(0), op == RW_READER);
67#define RW_LOCKED(rw, op) \
68 LOCKDEBUG_LOCKED(RW_DEBUG_P(rw), (rw), NULL, \
69 (uintptr_t)__builtin_return_address(0), op == RW_READER);
70#define RW_UNLOCKED(rw, op) \
71 LOCKDEBUG_UNLOCKED(RW_DEBUG_P(rw), (rw), \
72 (uintptr_t)__builtin_return_address(0), op == RW_READER);
73#define RW_DASSERT(rw, cond) \
74do { \
75 if (!(cond)) \
76 rw_abort(rw, __func__, "assertion failed: " #cond); \
77} while (/* CONSTCOND */ 0);
78
79#else /* LOCKDEBUG */
80
81#define RW_WANTLOCK(rw, op) /* nothing */
82#define RW_LOCKED(rw, op) /* nothing */
83#define RW_UNLOCKED(rw, op) /* nothing */
84#define RW_DASSERT(rw, cond) /* nothing */
85
86#endif /* LOCKDEBUG */
87
88/*
89 * DIAGNOSTIC
90 */
91
92#if defined(DIAGNOSTIC)
93
94#define RW_ASSERT(rw, cond) \
95do { \
96 if (!(cond)) \
97 rw_abort(rw, __func__, "assertion failed: " #cond); \
98} while (/* CONSTCOND */ 0)
99
100#else
101
102#define RW_ASSERT(rw, cond) /* nothing */
103
104#endif /* DIAGNOSTIC */
105
106#define RW_SETDEBUG(rw, on) ((rw)->rw_owner |= (on) ? 0 : RW_NODEBUG)
107#define RW_DEBUG_P(rw) (((rw)->rw_owner & RW_NODEBUG) == 0)
108#if defined(LOCKDEBUG)
109#define RW_INHERITDEBUG(n, o) (n) |= (o) & RW_NODEBUG
110#else /* defined(LOCKDEBUG) */
111#define RW_INHERITDEBUG(n, o) /* nothing */
112#endif /* defined(LOCKDEBUG) */
113
114static void rw_abort(krwlock_t *, const char *, const char *);
115static void rw_dump(volatile void *);
116static lwp_t *rw_owner(wchan_t);
117
118static inline uintptr_t
119rw_cas(krwlock_t *rw, uintptr_t o, uintptr_t n)
120{
121
122 RW_INHERITDEBUG(n, o);
123 return (uintptr_t)atomic_cas_ptr((volatile void *)&rw->rw_owner,
124 (void *)o, (void *)n);
125}
126
127static inline void
128rw_swap(krwlock_t *rw, uintptr_t o, uintptr_t n)
129{
130
131 RW_INHERITDEBUG(n, o);
132 n = (uintptr_t)atomic_swap_ptr((volatile void *)&rw->rw_owner,
133 (void *)n);
134 RW_DASSERT(rw, n == o);
135}
136
137/*
138 * For platforms that do not provide stubs, or for the LOCKDEBUG case.
139 */
140#ifdef LOCKDEBUG
141#undef __HAVE_RW_STUBS
142#endif
143
144#ifndef __HAVE_RW_STUBS
145__strong_alias(rw_enter,rw_vector_enter);
146__strong_alias(rw_exit,rw_vector_exit);
147__strong_alias(rw_tryenter,rw_vector_tryenter);
148#endif
149
150lockops_t rwlock_lockops = {
151 "Reader / writer lock",
152 LOCKOPS_SLEEP,
153 rw_dump
154};
155
156syncobj_t rw_syncobj = {
157 SOBJ_SLEEPQ_SORTED,
158 turnstile_unsleep,
159 turnstile_changepri,
160 sleepq_lendpri,
161 rw_owner,
162};
163
164/*
165 * rw_dump:
166 *
167 * Dump the contents of a rwlock structure.
168 */
169static void
170rw_dump(volatile void *cookie)
171{
172 volatile krwlock_t *rw = cookie;
173
174 printf_nolog("owner/count : %#018lx flags : %#018x\n",
175 (long)RW_OWNER(rw), (int)RW_FLAGS(rw));
176}
177
178/*
179 * rw_abort:
180 *
181 * Dump information about an error and panic the system. This
182 * generates a lot of machine code in the DIAGNOSTIC case, so
183 * we ask the compiler to not inline it.
184 */
185static void __noinline
186rw_abort(krwlock_t *rw, const char *func, const char *msg)
187{
188
189 if (panicstr != NULL)
190 return;
191
192 LOCKDEBUG_ABORT(rw, &rwlock_lockops, func, msg);
193}
194
195/*
196 * rw_init:
197 *
198 * Initialize a rwlock for use.
199 */
200void
201rw_init(krwlock_t *rw)
202{
203 bool dodebug;
204
205 memset(rw, 0, sizeof(*rw));
206
207 dodebug = LOCKDEBUG_ALLOC(rw, &rwlock_lockops,
208 (uintptr_t)__builtin_return_address(0));
209 RW_SETDEBUG(rw, dodebug);
210}
211
212/*
213 * rw_destroy:
214 *
215 * Tear down a rwlock.
216 */
217void
218rw_destroy(krwlock_t *rw)
219{
220
221 RW_ASSERT(rw, (rw->rw_owner & ~RW_NODEBUG) == 0);
222 LOCKDEBUG_FREE(RW_DEBUG_P(rw), rw);
223}
224
225/*
226 * rw_oncpu:
227 *
228 * Return true if an rwlock owner is running on a CPU in the system.
229 * If the target is waiting on the kernel big lock, then we must
230 * release it. This is necessary to avoid deadlock.
231 */
232static bool
233rw_oncpu(uintptr_t owner)
234{
235#ifdef MULTIPROCESSOR
236 struct cpu_info *ci;
237 lwp_t *l;
238
239 KASSERT(kpreempt_disabled());
240
241 if ((owner & (RW_WRITE_LOCKED|RW_HAS_WAITERS)) != RW_WRITE_LOCKED) {
242 return false;
243 }
244
245 /*
246 * See lwp_dtor() why dereference of the LWP pointer is safe.
247 * We must have kernel preemption disabled for that.
248 */
249 l = (lwp_t *)(owner & RW_THREAD);
250 ci = l->l_cpu;
251
252 if (ci && ci->ci_curlwp == l) {
253 /* Target is running; do we need to block? */
254 return (ci->ci_biglock_wanted != l);
255 }
256#endif
257 /* Not running. It may be safe to block now. */
258 return false;
259}
260
261/*
262 * rw_vector_enter:
263 *
264 * Acquire a rwlock.
265 */
266void
267rw_vector_enter(krwlock_t *rw, const krw_t op)
268{
269 uintptr_t owner, incr, need_wait, set_wait, curthread, next;
270 turnstile_t *ts;
271 int queue;
272 lwp_t *l;
273 LOCKSTAT_TIMER(slptime);
274 LOCKSTAT_TIMER(slpcnt);
275 LOCKSTAT_TIMER(spintime);
276 LOCKSTAT_COUNTER(spincnt);
277 LOCKSTAT_FLAG(lsflag);
278
279 l = curlwp;
280 curthread = (uintptr_t)l;
281
282 RW_ASSERT(rw, !cpu_intr_p());
283 RW_ASSERT(rw, curthread != 0);
284 RW_WANTLOCK(rw, op);
285
286 if (panicstr == NULL) {
287 LOCKDEBUG_BARRIER(&kernel_lock, 1);
288 }
289
290 /*
291 * We play a slight trick here. If we're a reader, we want
292 * increment the read count. If we're a writer, we want to
293 * set the owner field and the WRITE_LOCKED bit.
294 *
295 * In the latter case, we expect those bits to be zero,
296 * therefore we can use an add operation to set them, which
297 * means an add operation for both cases.
298 */
299 if (__predict_true(op == RW_READER)) {
300 incr = RW_READ_INCR;
301 set_wait = RW_HAS_WAITERS;
302 need_wait = RW_WRITE_LOCKED | RW_WRITE_WANTED;
303 queue = TS_READER_Q;
304 } else {
305 RW_DASSERT(rw, op == RW_WRITER);
306 incr = curthread | RW_WRITE_LOCKED;
307 set_wait = RW_HAS_WAITERS | RW_WRITE_WANTED;
308 need_wait = RW_WRITE_LOCKED | RW_THREAD;
309 queue = TS_WRITER_Q;
310 }
311
312 LOCKSTAT_ENTER(lsflag);
313
314 KPREEMPT_DISABLE(curlwp);
315 for (owner = rw->rw_owner; ;) {
316 /*
317 * Read the lock owner field. If the need-to-wait
318 * indicator is clear, then try to acquire the lock.
319 */
320 if ((owner & need_wait) == 0) {
321 next = rw_cas(rw, owner, (owner + incr) &
322 ~RW_WRITE_WANTED);
323 if (__predict_true(next == owner)) {
324 /* Got it! */
325 membar_enter();
326 break;
327 }
328
329 /*
330 * Didn't get it -- spin around again (we'll
331 * probably sleep on the next iteration).
332 */
333 owner = next;
334 continue;
335 }
336 if (__predict_false(panicstr != NULL)) {
337 KPREEMPT_ENABLE(curlwp);
338 return;
339 }
340 if (__predict_false(RW_OWNER(rw) == curthread)) {
341 rw_abort(rw, __func__, "locking against myself");
342 }
343 /*
344 * If the lock owner is running on another CPU, and
345 * there are no existing waiters, then spin.
346 */
347 if (rw_oncpu(owner)) {
348 LOCKSTAT_START_TIMER(lsflag, spintime);
349 u_int count = SPINLOCK_BACKOFF_MIN;
350 do {
351 KPREEMPT_ENABLE(curlwp);
352 SPINLOCK_BACKOFF(count);
353 KPREEMPT_DISABLE(curlwp);
354 owner = rw->rw_owner;
355 } while (rw_oncpu(owner));
356 LOCKSTAT_STOP_TIMER(lsflag, spintime);
357 LOCKSTAT_COUNT(spincnt, 1);
358 if ((owner & need_wait) == 0)
359 continue;
360 }
361
362 /*
363 * Grab the turnstile chain lock. Once we have that, we
364 * can adjust the waiter bits and sleep queue.
365 */
366 ts = turnstile_lookup(rw);
367
368 /*
369 * Mark the rwlock as having waiters. If the set fails,
370 * then we may not need to sleep and should spin again.
371 * Reload rw_owner because turnstile_lookup() may have
372 * spun on the turnstile chain lock.
373 */
374 owner = rw->rw_owner;
375 if ((owner & need_wait) == 0 || rw_oncpu(owner)) {
376 turnstile_exit(rw);
377 continue;
378 }
379 next = rw_cas(rw, owner, owner | set_wait);
380 if (__predict_false(next != owner)) {
381 turnstile_exit(rw);
382 owner = next;
383 continue;
384 }
385
386 LOCKSTAT_START_TIMER(lsflag, slptime);
387 turnstile_block(ts, queue, rw, &rw_syncobj);
388 LOCKSTAT_STOP_TIMER(lsflag, slptime);
389 LOCKSTAT_COUNT(slpcnt, 1);
390
391 /*
392 * No need for a memory barrier because of context switch.
393 * If not handed the lock, then spin again.
394 */
395 if (op == RW_READER || (rw->rw_owner & RW_THREAD) == curthread)
396 break;
397
398 owner = rw->rw_owner;
399 }
400 KPREEMPT_ENABLE(curlwp);
401
402 LOCKSTAT_EVENT(lsflag, rw, LB_RWLOCK |
403 (op == RW_WRITER ? LB_SLEEP1 : LB_SLEEP2), slpcnt, slptime);
404 LOCKSTAT_EVENT(lsflag, rw, LB_RWLOCK | LB_SPIN, spincnt, spintime);
405 LOCKSTAT_EXIT(lsflag);
406
407 RW_DASSERT(rw, (op != RW_READER && RW_OWNER(rw) == curthread) ||
408 (op == RW_READER && RW_COUNT(rw) != 0));
409 RW_LOCKED(rw, op);
410}
411
412/*
413 * rw_vector_exit:
414 *
415 * Release a rwlock.
416 */
417void
418rw_vector_exit(krwlock_t *rw)
419{
420 uintptr_t curthread, owner, decr, newown, next;
421 turnstile_t *ts;
422 int rcnt, wcnt;
423 lwp_t *l;
424
425 curthread = (uintptr_t)curlwp;
426 RW_ASSERT(rw, curthread != 0);
427
428 if (__predict_false(panicstr != NULL))
429 return;
430
431 /*
432 * Again, we use a trick. Since we used an add operation to
433 * set the required lock bits, we can use a subtract to clear
434 * them, which makes the read-release and write-release path
435 * the same.
436 */
437 owner = rw->rw_owner;
438 if (__predict_false((owner & RW_WRITE_LOCKED) != 0)) {
439 RW_UNLOCKED(rw, RW_WRITER);
440 RW_ASSERT(rw, RW_OWNER(rw) == curthread);
441 decr = curthread | RW_WRITE_LOCKED;
442 } else {
443 RW_UNLOCKED(rw, RW_READER);
444 RW_ASSERT(rw, RW_COUNT(rw) != 0);
445 decr = RW_READ_INCR;
446 }
447
448 /*
449 * Compute what we expect the new value of the lock to be. Only
450 * proceed to do direct handoff if there are waiters, and if the
451 * lock would become unowned.
452 */
453 membar_exit();
454 for (;;) {
455 newown = (owner - decr);
456 if ((newown & (RW_THREAD | RW_HAS_WAITERS)) == RW_HAS_WAITERS)
457 break;
458 next = rw_cas(rw, owner, newown);
459 if (__predict_true(next == owner))
460 return;
461 owner = next;
462 }
463
464 /*
465 * Grab the turnstile chain lock. This gets the interlock
466 * on the sleep queue. Once we have that, we can adjust the
467 * waiter bits.
468 */
469 ts = turnstile_lookup(rw);
470 owner = rw->rw_owner;
471 RW_DASSERT(rw, ts != NULL);
472 RW_DASSERT(rw, (owner & RW_HAS_WAITERS) != 0);
473
474 wcnt = TS_WAITERS(ts, TS_WRITER_Q);
475 rcnt = TS_WAITERS(ts, TS_READER_Q);
476
477 /*
478 * Give the lock away.
479 *
480 * If we are releasing a write lock, then prefer to wake all
481 * outstanding readers. Otherwise, wake one writer if there
482 * are outstanding readers, or all writers if there are no
483 * pending readers. If waking one specific writer, the writer
484 * is handed the lock here. If waking multiple writers, we
485 * set WRITE_WANTED to block out new readers, and let them
486 * do the work of acquiring the lock in rw_vector_enter().
487 */
488 if (rcnt == 0 || decr == RW_READ_INCR) {
489 RW_DASSERT(rw, wcnt != 0);
490 RW_DASSERT(rw, (owner & RW_WRITE_WANTED) != 0);
491
492 if (rcnt != 0) {
493 /* Give the lock to the longest waiting writer. */
494 l = TS_FIRST(ts, TS_WRITER_Q);
495 newown = (uintptr_t)l | RW_WRITE_LOCKED | RW_HAS_WAITERS;
496 if (wcnt > 1)
497 newown |= RW_WRITE_WANTED;
498 rw_swap(rw, owner, newown);
499 turnstile_wakeup(ts, TS_WRITER_Q, 1, l);
500 } else {
501 /* Wake all writers and let them fight it out. */
502 rw_swap(rw, owner, RW_WRITE_WANTED);
503 turnstile_wakeup(ts, TS_WRITER_Q, wcnt, NULL);
504 }
505 } else {
506 RW_DASSERT(rw, rcnt != 0);
507
508 /*
509 * Give the lock to all blocked readers. If there
510 * is a writer waiting, new readers that arrive
511 * after the release will be blocked out.
512 */
513 newown = rcnt << RW_READ_COUNT_SHIFT;
514 if (wcnt != 0)
515 newown |= RW_HAS_WAITERS | RW_WRITE_WANTED;
516
517 /* Wake up all sleeping readers. */
518 rw_swap(rw, owner, newown);
519 turnstile_wakeup(ts, TS_READER_Q, rcnt, NULL);
520 }
521}
522
523/*
524 * rw_vector_tryenter:
525 *
526 * Try to acquire a rwlock.
527 */
528int
529rw_vector_tryenter(krwlock_t *rw, const krw_t op)
530{
531 uintptr_t curthread, owner, incr, need_wait, next;
532
533 curthread = (uintptr_t)curlwp;
534
535 RW_ASSERT(rw, curthread != 0);
536
537 if (op == RW_READER) {
538 incr = RW_READ_INCR;
539 need_wait = RW_WRITE_LOCKED | RW_WRITE_WANTED;
540 } else {
541 RW_DASSERT(rw, op == RW_WRITER);
542 incr = curthread | RW_WRITE_LOCKED;
543 need_wait = RW_WRITE_LOCKED | RW_THREAD;
544 }
545
546 for (owner = rw->rw_owner;; owner = next) {
547 owner = rw->rw_owner;
548 if (__predict_false((owner & need_wait) != 0))
549 return 0;
550 next = rw_cas(rw, owner, owner + incr);
551 if (__predict_true(next == owner)) {
552 /* Got it! */
553 membar_enter();
554 break;
555 }
556 }
557
558 RW_WANTLOCK(rw, op);
559 RW_LOCKED(rw, op);
560 RW_DASSERT(rw, (op != RW_READER && RW_OWNER(rw) == curthread) ||
561 (op == RW_READER && RW_COUNT(rw) != 0));
562
563 return 1;
564}
565
566/*
567 * rw_downgrade:
568 *
569 * Downgrade a write lock to a read lock.
570 */
571void
572rw_downgrade(krwlock_t *rw)
573{
574 uintptr_t owner, curthread, newown, next;
575 turnstile_t *ts;
576 int rcnt, wcnt;
577
578 curthread = (uintptr_t)curlwp;
579 RW_ASSERT(rw, curthread != 0);
580 RW_DASSERT(rw, (rw->rw_owner & RW_WRITE_LOCKED) != 0);
581 RW_ASSERT(rw, RW_OWNER(rw) == curthread);
582 RW_UNLOCKED(rw, RW_WRITER);
583#if !defined(DIAGNOSTIC)
584 __USE(curthread);
585#endif
586
587
588 membar_producer();
589 owner = rw->rw_owner;
590 if ((owner & RW_HAS_WAITERS) == 0) {
591 /*
592 * There are no waiters, so we can do this the easy way.
593 * Try swapping us down to one read hold. If it fails, the
594 * lock condition has changed and we most likely now have
595 * waiters.
596 */
597 next = rw_cas(rw, owner, RW_READ_INCR);
598 if (__predict_true(next == owner)) {
599 RW_LOCKED(rw, RW_READER);
600 RW_DASSERT(rw, (rw->rw_owner & RW_WRITE_LOCKED) == 0);
601 RW_DASSERT(rw, RW_COUNT(rw) != 0);
602 return;
603 }
604 owner = next;
605 }
606
607 /*
608 * Grab the turnstile chain lock. This gets the interlock
609 * on the sleep queue. Once we have that, we can adjust the
610 * waiter bits.
611 */
612 for (;; owner = next) {
613 ts = turnstile_lookup(rw);
614 RW_DASSERT(rw, ts != NULL);
615
616 rcnt = TS_WAITERS(ts, TS_READER_Q);
617 wcnt = TS_WAITERS(ts, TS_WRITER_Q);
618
619 /*
620 * If there are no readers, just preserve the waiters
621 * bits, swap us down to one read hold and return.
622 */
623 if (rcnt == 0) {
624 RW_DASSERT(rw, wcnt != 0);
625 RW_DASSERT(rw, (rw->rw_owner & RW_WRITE_WANTED) != 0);
626 RW_DASSERT(rw, (rw->rw_owner & RW_HAS_WAITERS) != 0);
627
628 newown = RW_READ_INCR | RW_HAS_WAITERS | RW_WRITE_WANTED;
629 next = rw_cas(rw, owner, newown);
630 turnstile_exit(rw);
631 if (__predict_true(next == owner))
632 break;
633 } else {
634 /*
635 * Give the lock to all blocked readers. We may
636 * retain one read hold if downgrading. If there
637 * is a writer waiting, new readers will be blocked
638 * out.
639 */
640 newown = (rcnt << RW_READ_COUNT_SHIFT) + RW_READ_INCR;
641 if (wcnt != 0)
642 newown |= RW_HAS_WAITERS | RW_WRITE_WANTED;
643
644 next = rw_cas(rw, owner, newown);
645 if (__predict_true(next == owner)) {
646 /* Wake up all sleeping readers. */
647 turnstile_wakeup(ts, TS_READER_Q, rcnt, NULL);
648 break;
649 }
650 turnstile_exit(rw);
651 }
652 }
653
654 RW_WANTLOCK(rw, RW_READER);
655 RW_LOCKED(rw, RW_READER);
656 RW_DASSERT(rw, (rw->rw_owner & RW_WRITE_LOCKED) == 0);
657 RW_DASSERT(rw, RW_COUNT(rw) != 0);
658}
659
660/*
661 * rw_tryupgrade:
662 *
663 * Try to upgrade a read lock to a write lock. We must be the
664 * only reader.
665 */
666int
667rw_tryupgrade(krwlock_t *rw)
668{
669 uintptr_t owner, curthread, newown, next;
670
671 curthread = (uintptr_t)curlwp;
672 RW_ASSERT(rw, curthread != 0);
673 RW_ASSERT(rw, rw_read_held(rw));
674
675 for (owner = rw->rw_owner;; owner = next) {
676 RW_ASSERT(rw, (owner & RW_WRITE_LOCKED) == 0);
677 if (__predict_false((owner & RW_THREAD) != RW_READ_INCR)) {
678 RW_ASSERT(rw, (owner & RW_THREAD) != 0);
679 return 0;
680 }
681 newown = curthread | RW_WRITE_LOCKED | (owner & ~RW_THREAD);
682 next = rw_cas(rw, owner, newown);
683 if (__predict_true(next == owner)) {
684 membar_producer();
685 break;
686 }
687 }
688
689 RW_UNLOCKED(rw, RW_READER);
690 RW_WANTLOCK(rw, RW_WRITER);
691 RW_LOCKED(rw, RW_WRITER);
692 RW_DASSERT(rw, rw->rw_owner & RW_WRITE_LOCKED);
693 RW_DASSERT(rw, RW_OWNER(rw) == curthread);
694
695 return 1;
696}
697
698/*
699 * rw_read_held:
700 *
701 * Returns true if the rwlock is held for reading. Must only be
702 * used for diagnostic assertions, and never be used to make
703 * decisions about how to use a rwlock.
704 */
705int
706rw_read_held(krwlock_t *rw)
707{
708 uintptr_t owner;
709
710 if (panicstr != NULL)
711 return 1;
712 if (rw == NULL)
713 return 0;
714 owner = rw->rw_owner;
715 return (owner & RW_WRITE_LOCKED) == 0 && (owner & RW_THREAD) != 0;
716}
717
718/*
719 * rw_write_held:
720 *
721 * Returns true if the rwlock is held for writing. Must only be
722 * used for diagnostic assertions, and never be used to make
723 * decisions about how to use a rwlock.
724 */
725int
726rw_write_held(krwlock_t *rw)
727{
728
729 if (panicstr != NULL)
730 return 1;
731 if (rw == NULL)
732 return 0;
733 return (rw->rw_owner & (RW_WRITE_LOCKED | RW_THREAD)) ==
734 (RW_WRITE_LOCKED | (uintptr_t)curlwp);
735}
736
737/*
738 * rw_lock_held:
739 *
740 * Returns true if the rwlock is held for reading or writing. Must
741 * only be used for diagnostic assertions, and never be used to make
742 * decisions about how to use a rwlock.
743 */
744int
745rw_lock_held(krwlock_t *rw)
746{
747
748 if (panicstr != NULL)
749 return 1;
750 if (rw == NULL)
751 return 0;
752 return (rw->rw_owner & RW_THREAD) != 0;
753}
754
755/*
756 * rw_owner:
757 *
758 * Return the current owner of an RW lock, but only if it is write
759 * held. Used for priority inheritance.
760 */
761static lwp_t *
762rw_owner(wchan_t obj)
763{
764 krwlock_t *rw = (void *)(uintptr_t)obj; /* discard qualifiers */
765 uintptr_t owner = rw->rw_owner;
766
767 if ((owner & RW_WRITE_LOCKED) == 0)
768 return NULL;
769
770 return (void *)(owner & RW_THREAD);
771}
772