1/* $NetBSD: sleepq.h,v 1.24 2015/02/08 19:39:09 christos Exp $ */
2
3/*-
4 * Copyright (c) 2002, 2006, 2007, 2008, 2009 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Jason R. Thorpe and Andrew Doran.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32#ifndef _SYS_SLEEPQ_H_
33#define _SYS_SLEEPQ_H_
34
35#include <sys/lwp.h>
36#include <sys/mutex.h>
37#include <sys/pool.h>
38#include <sys/queue.h>
39#include <sys/sched.h>
40#include <sys/syncobj.h>
41
42/*
43 * Generic sleep queues.
44 */
45
46#define SLEEPTAB_HASH_SHIFT 7
47#define SLEEPTAB_HASH_SIZE (1 << SLEEPTAB_HASH_SHIFT)
48#define SLEEPTAB_HASH_MASK (SLEEPTAB_HASH_SIZE - 1)
49#define SLEEPTAB_HASH(wchan) (((uintptr_t)(wchan) >> 8) & SLEEPTAB_HASH_MASK)
50
51TAILQ_HEAD(sleepq, lwp);
52
53typedef struct sleepq sleepq_t;
54
55typedef struct sleeptab {
56 struct {
57 kmutex_t *st_mutex;
58 sleepq_t st_queue;
59 } st_queues[SLEEPTAB_HASH_SIZE];
60} sleeptab_t;
61
62void sleepq_init(sleepq_t *);
63void sleepq_remove(sleepq_t *, lwp_t *);
64void sleepq_enqueue(sleepq_t *, wchan_t, const char *, struct syncobj *);
65void sleepq_unsleep(lwp_t *, bool);
66void sleepq_timeout(void *);
67void sleepq_wake(sleepq_t *, wchan_t, u_int, kmutex_t *);
68int sleepq_abort(kmutex_t *, int);
69void sleepq_changepri(lwp_t *, pri_t);
70void sleepq_lendpri(lwp_t *, pri_t);
71int sleepq_block(int, bool);
72
73void sleeptab_init(sleeptab_t *);
74
75extern sleeptab_t sleeptab;
76
77#ifdef _KERNEL
78/*
79 * Return non-zero if it is unsafe to sleep.
80 *
81 * XXX This only exists because panic() is broken.
82 */
83static inline bool
84sleepq_dontsleep(lwp_t *l)
85{
86 extern int cold;
87
88 return cold || (doing_shutdown && (panicstr || CURCPU_IDLE_P()));
89}
90
91/*
92 * Find the correct sleep queue for the specified wait channel. This
93 * acquires and holds the per-queue interlock.
94 */
95static inline sleepq_t *
96sleeptab_lookup(sleeptab_t *st, wchan_t wchan, kmutex_t **mp)
97{
98 sleepq_t *sq;
99
100 sq = &st->st_queues[SLEEPTAB_HASH(wchan)].st_queue;
101 *mp = st->st_queues[SLEEPTAB_HASH(wchan)].st_mutex;
102 mutex_spin_enter(*mp);
103 return sq;
104}
105
106static inline kmutex_t *
107sleepq_hashlock(wchan_t wchan)
108{
109 kmutex_t *mp;
110
111 mp = sleeptab.st_queues[SLEEPTAB_HASH(wchan)].st_mutex;
112 mutex_spin_enter(mp);
113 return mp;
114}
115
116/*
117 * Prepare to block on a sleep queue, after which any interlock can be
118 * safely released.
119 */
120static inline void
121sleepq_enter(sleepq_t *sq, lwp_t *l, kmutex_t *mp)
122{
123
124 /*
125 * Acquire the per-LWP mutex and lend it ours sleep queue lock.
126 * Once interlocked, we can release the kernel lock.
127 */
128 lwp_lock(l);
129 lwp_unlock_to(l, mp);
130 KERNEL_UNLOCK_ALL(NULL, &l->l_biglocks);
131}
132#endif
133
134/*
135 * Turnstiles, specialized sleep queues for use by kernel locks.
136 */
137
138typedef struct turnstile {
139 LIST_ENTRY(turnstile) ts_chain; /* link on hash chain */
140 struct turnstile *ts_free; /* turnstile free list */
141 wchan_t ts_obj; /* lock object */
142 sleepq_t ts_sleepq[2]; /* sleep queues */
143 u_int ts_waiters[2]; /* count of waiters */
144
145 /* priority inheritance */
146 pri_t ts_eprio;
147 lwp_t *ts_inheritor;
148 SLIST_ENTRY(turnstile) ts_pichain;
149} turnstile_t;
150
151typedef struct tschain {
152 kmutex_t *tc_mutex; /* mutex on structs & queues */
153 LIST_HEAD(, turnstile) tc_chain; /* turnstile chain */
154} tschain_t;
155
156#define TS_READER_Q 0 /* reader sleep queue */
157#define TS_WRITER_Q 1 /* writer sleep queue */
158
159#define TS_WAITERS(ts, q) \
160 (ts)->ts_waiters[(q)]
161
162#define TS_ALL_WAITERS(ts) \
163 ((ts)->ts_waiters[TS_READER_Q] + \
164 (ts)->ts_waiters[TS_WRITER_Q])
165
166#define TS_FIRST(ts, q) (TAILQ_FIRST(&(ts)->ts_sleepq[(q)]))
167
168#ifdef _KERNEL
169
170void turnstile_init(void);
171turnstile_t *turnstile_lookup(wchan_t);
172void turnstile_exit(wchan_t);
173void turnstile_block(turnstile_t *, int, wchan_t, syncobj_t *);
174void turnstile_wakeup(turnstile_t *, int, int, lwp_t *);
175void turnstile_print(volatile void *, void (*)(const char *, ...)
176 __printflike(1, 2));
177void turnstile_unsleep(lwp_t *, bool);
178void turnstile_changepri(lwp_t *, pri_t);
179
180extern pool_cache_t turnstile_cache;
181extern turnstile_t turnstile0;
182
183#endif /* _KERNEL */
184
185#endif /* _SYS_SLEEPQ_H_ */
186