1/* $NetBSD: subr_pcu.c,v 1.19 2014/05/25 14:53:55 rmind Exp $ */
2
3/*-
4 * Copyright (c) 2011, 2014 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Mindaugas Rasiukevicius.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32/*
33 * Per CPU Unit (PCU) - is an interface to manage synchronization of any
34 * per CPU context (unit) tied with LWP context. Typical use: FPU state.
35 *
36 * Concurrency notes:
37 *
38 * PCU state may be loaded only by the current LWP, that is, curlwp.
39 * Therefore, only LWP itself can set a CPU for lwp_t::l_pcu_cpu[id].
40 *
41 * There are some important rules about operation calls. The request
42 * for a PCU release can be from a) the owner LWP (regardless whether
43 * the PCU state is on the current CPU or remote CPU) b) any other LWP
44 * running on that CPU (in such case, the owner LWP is on a remote CPU
45 * or sleeping).
46 *
47 * In any case, the PCU state can *only* be changed from the current
48 * CPU. If said PCU state is on the remote CPU, a cross-call will be
49 * sent by the owner LWP. Therefore struct cpu_info::ci_pcu_curlwp[id]
50 * may only be changed by the current CPU and lwp_t::l_pcu_cpu[id] may
51 * only be cleared by the CPU which has the PCU state loaded.
52 */
53
54#include <sys/cdefs.h>
55__KERNEL_RCSID(0, "$NetBSD: subr_pcu.c,v 1.19 2014/05/25 14:53:55 rmind Exp $");
56
57#include <sys/param.h>
58#include <sys/cpu.h>
59#include <sys/lwp.h>
60#include <sys/pcu.h>
61#include <sys/ipi.h>
62
63#if PCU_UNIT_COUNT > 0
64
65static inline void pcu_do_op(const pcu_ops_t *, lwp_t * const, const int);
66static void pcu_lwp_op(const pcu_ops_t *, lwp_t *, const int);
67
68/*
69 * Internal PCU commands for the pcu_do_op() function.
70 */
71#define PCU_CMD_SAVE 0x01 /* save PCU state to the LWP */
72#define PCU_CMD_RELEASE 0x02 /* release PCU state on the CPU */
73
74/*
75 * Message structure for another CPU passed via ipi(9).
76 */
77typedef struct {
78 const pcu_ops_t *pcu;
79 lwp_t * owner;
80 const int flags;
81} pcu_ipi_msg_t;
82
83/*
84 * PCU IPIs run at IPL_HIGH (aka IPL_PCU in this code).
85 */
86#define splpcu splhigh
87
88/* PCU operations structure provided by the MD code. */
89extern const pcu_ops_t * const pcu_ops_md_defs[];
90
91/*
92 * pcu_switchpoint: release PCU state if the LWP is being run on another CPU.
93 * This routine is called on each context switch by by mi_switch().
94 */
95void
96pcu_switchpoint(lwp_t *l)
97{
98 const uint32_t pcu_valid = l->l_pcu_valid;
99 int s;
100
101 KASSERTMSG(l == curlwp, "l %p != curlwp %p", l, curlwp);
102
103 if (__predict_true(pcu_valid == 0)) {
104 /* PCUs are not in use. */
105 return;
106 }
107 s = splpcu();
108 for (u_int id = 0; id < PCU_UNIT_COUNT; id++) {
109 if ((pcu_valid & (1U << id)) == 0) {
110 continue;
111 }
112 struct cpu_info * const pcu_ci = l->l_pcu_cpu[id];
113 if (pcu_ci == NULL || pcu_ci == l->l_cpu) {
114 continue;
115 }
116 const pcu_ops_t * const pcu = pcu_ops_md_defs[id];
117 pcu->pcu_state_release(l);
118 }
119 splx(s);
120}
121
122/*
123 * pcu_discard_all: discard PCU state of the given LWP.
124 *
125 * Used by exec and LWP exit.
126 */
127void
128pcu_discard_all(lwp_t *l)
129{
130 const uint32_t pcu_valid = l->l_pcu_valid;
131
132 KASSERT(l == curlwp || ((l->l_flag & LW_SYSTEM) && pcu_valid == 0));
133
134 if (__predict_true(pcu_valid == 0)) {
135 /* PCUs are not in use. */
136 return;
137 }
138 for (u_int id = 0; id < PCU_UNIT_COUNT; id++) {
139 if ((pcu_valid & (1U << id)) == 0) {
140 continue;
141 }
142 if (__predict_true(l->l_pcu_cpu[id] == NULL)) {
143 continue;
144 }
145 const pcu_ops_t * const pcu = pcu_ops_md_defs[id];
146 pcu_lwp_op(pcu, l, PCU_CMD_RELEASE);
147 }
148 l->l_pcu_valid = 0;
149}
150
151/*
152 * pcu_save_all: save PCU state of the given LWP so that eg. coredump can
153 * examine it.
154 */
155void
156pcu_save_all(lwp_t *l)
157{
158 const uint32_t pcu_valid = l->l_pcu_valid;
159 int flags = PCU_CMD_SAVE;
160
161 /* If LW_WCORE, we are also releasing the state. */
162 if (__predict_false(l->l_flag & LW_WCORE)) {
163 flags |= PCU_CMD_RELEASE;
164 }
165
166 /*
167 * Normally we save for the current LWP, but sometimes we get called
168 * with a different LWP (forking a system LWP or doing a coredump of
169 * a process with multiple threads) and we need to deal with that.
170 */
171 KASSERT(l == curlwp || (((l->l_flag & LW_SYSTEM) ||
172 (curlwp->l_proc == l->l_proc && l->l_stat == LSSUSPENDED)) &&
173 pcu_valid == 0));
174
175 if (__predict_true(pcu_valid == 0)) {
176 /* PCUs are not in use. */
177 return;
178 }
179 for (u_int id = 0; id < PCU_UNIT_COUNT; id++) {
180 if ((pcu_valid & (1U << id)) == 0) {
181 continue;
182 }
183 if (__predict_true(l->l_pcu_cpu[id] == NULL)) {
184 continue;
185 }
186 const pcu_ops_t * const pcu = pcu_ops_md_defs[id];
187 pcu_lwp_op(pcu, l, flags);
188 }
189}
190
191/*
192 * pcu_do_op: save/release PCU state on the current CPU.
193 *
194 * => Must be called at IPL_PCU or from the interrupt.
195 */
196static inline void
197pcu_do_op(const pcu_ops_t *pcu, lwp_t * const l, const int flags)
198{
199 struct cpu_info * const ci = curcpu();
200 const u_int id = pcu->pcu_id;
201
202 KASSERT(l->l_pcu_cpu[id] == ci);
203
204 if (flags & PCU_CMD_SAVE) {
205 pcu->pcu_state_save(l);
206 }
207 if (flags & PCU_CMD_RELEASE) {
208 pcu->pcu_state_release(l);
209 ci->ci_pcu_curlwp[id] = NULL;
210 l->l_pcu_cpu[id] = NULL;
211 }
212}
213
214/*
215 * pcu_cpu_ipi: helper routine to call pcu_do_op() via ipi(9).
216 */
217static void
218pcu_cpu_ipi(void *arg)
219{
220 const pcu_ipi_msg_t *pcu_msg = arg;
221 const pcu_ops_t *pcu = pcu_msg->pcu;
222 const u_int id = pcu->pcu_id;
223 lwp_t *l = pcu_msg->owner;
224
225 KASSERT(pcu_msg->owner != NULL);
226
227 if (curcpu()->ci_pcu_curlwp[id] != l) {
228 /*
229 * Different ownership: another LWP raced with us and
230 * perform save and release. There is nothing to do.
231 */
232 KASSERT(l->l_pcu_cpu[id] == NULL);
233 return;
234 }
235 pcu_do_op(pcu, l, pcu_msg->flags);
236}
237
238/*
239 * pcu_lwp_op: perform PCU state save, release or both operations on LWP.
240 */
241static void
242pcu_lwp_op(const pcu_ops_t *pcu, lwp_t *l, const int flags)
243{
244 const u_int id = pcu->pcu_id;
245 struct cpu_info *ci;
246 int s;
247
248 /*
249 * Caller should have re-checked if there is any state to manage.
250 * Block the interrupts and inspect again, since cross-call sent
251 * by remote CPU could have changed the state.
252 */
253 s = splpcu();
254 ci = l->l_pcu_cpu[id];
255 if (ci == curcpu()) {
256 /*
257 * State is on the current CPU - just perform the operations.
258 */
259 KASSERTMSG(ci->ci_pcu_curlwp[id] == l,
260 "%s: cpu%u: pcu_curlwp[%u] (%p) != l (%p)",
261 __func__, cpu_index(ci), id, ci->ci_pcu_curlwp[id], l);
262 pcu_do_op(pcu, l, flags);
263 splx(s);
264 return;
265 }
266 if (__predict_false(ci == NULL)) {
267 /* Cross-call has won the race - no state to manage. */
268 splx(s);
269 return;
270 }
271
272 /*
273 * The state is on the remote CPU: perform the operation(s) there.
274 */
275 pcu_ipi_msg_t pcu_msg = { .pcu = pcu, .owner = l, .flags = flags };
276 ipi_msg_t ipi_msg = { .func = pcu_cpu_ipi, .arg = &pcu_msg };
277 ipi_unicast(&ipi_msg, ci);
278 splx(s);
279
280 /* Wait for completion. */
281 ipi_wait(&ipi_msg);
282
283 KASSERT((flags & PCU_CMD_RELEASE) == 0 || l->l_pcu_cpu[id] == NULL);
284}
285
286/*
287 * pcu_load: load/initialize the PCU state of current LWP on current CPU.
288 */
289void
290pcu_load(const pcu_ops_t *pcu)
291{
292 lwp_t *oncpu_lwp, * const l = curlwp;
293 const u_int id = pcu->pcu_id;
294 struct cpu_info *ci, *curci;
295 int s;
296
297 KASSERT(!cpu_intr_p() && !cpu_softintr_p());
298
299 s = splpcu();
300 curci = curcpu();
301 ci = l->l_pcu_cpu[id];
302
303 /* Does this CPU already have our PCU state loaded? */
304 if (ci == curci) {
305 /*
306 * Fault reoccurred while the PCU state is loaded and
307 * therefore PCU should be re‐enabled. This happens
308 * if LWP is context switched to another CPU and then
309 * switched back to the original CPU while the state
310 * on that CPU has not been changed by other LWPs.
311 *
312 * It may also happen due to instruction "bouncing" on
313 * some architectures.
314 */
315 KASSERT(curci->ci_pcu_curlwp[id] == l);
316 KASSERT(pcu_valid_p(pcu));
317 pcu->pcu_state_load(l, PCU_VALID | PCU_REENABLE);
318 splx(s);
319 return;
320 }
321
322 /* If PCU state of this LWP is on the remote CPU - save it there. */
323 if (ci) {
324 pcu_ipi_msg_t pcu_msg = { .pcu = pcu, .owner = l,
325 .flags = PCU_CMD_SAVE | PCU_CMD_RELEASE };
326 ipi_msg_t ipi_msg = { .func = pcu_cpu_ipi, .arg = &pcu_msg };
327 ipi_unicast(&ipi_msg, ci);
328 splx(s);
329
330 /*
331 * Wait for completion, re-enter IPL_PCU and re-fetch
332 * the current CPU.
333 */
334 ipi_wait(&ipi_msg);
335 s = splpcu();
336 curci = curcpu();
337 }
338 KASSERT(l->l_pcu_cpu[id] == NULL);
339
340 /* Save the PCU state on the current CPU, if there is any. */
341 if ((oncpu_lwp = curci->ci_pcu_curlwp[id]) != NULL) {
342 pcu_do_op(pcu, oncpu_lwp, PCU_CMD_SAVE | PCU_CMD_RELEASE);
343 KASSERT(curci->ci_pcu_curlwp[id] == NULL);
344 }
345
346 /*
347 * Finally, load the state for this LWP on this CPU. Indicate to
348 * the load function whether PCU state was valid before this call.
349 */
350 const bool valid = ((1U << id) & l->l_pcu_valid) != 0;
351 pcu->pcu_state_load(l, valid ? PCU_VALID : 0);
352 curci->ci_pcu_curlwp[id] = l;
353 l->l_pcu_cpu[id] = curci;
354 l->l_pcu_valid |= (1U << id);
355 splx(s);
356}
357
358/*
359 * pcu_discard: discard the PCU state of current LWP. If "valid"
360 * parameter is true, then keep considering the PCU state as valid.
361 */
362void
363pcu_discard(const pcu_ops_t *pcu, bool valid)
364{
365 const u_int id = pcu->pcu_id;
366 lwp_t * const l = curlwp;
367
368 KASSERT(!cpu_intr_p() && !cpu_softintr_p());
369
370 if (__predict_false(valid)) {
371 l->l_pcu_valid |= (1U << id);
372 } else {
373 l->l_pcu_valid &= ~(1U << id);
374 }
375 if (__predict_true(l->l_pcu_cpu[id] == NULL)) {
376 return;
377 }
378 pcu_lwp_op(pcu, l, PCU_CMD_RELEASE);
379}
380
381/*
382 * pcu_save_lwp: save PCU state to the given LWP.
383 */
384void
385pcu_save(const pcu_ops_t *pcu)
386{
387 const u_int id = pcu->pcu_id;
388 lwp_t * const l = curlwp;
389
390 KASSERT(!cpu_intr_p() && !cpu_softintr_p());
391
392 if (__predict_true(l->l_pcu_cpu[id] == NULL)) {
393 return;
394 }
395 pcu_lwp_op(pcu, l, PCU_CMD_SAVE | PCU_CMD_RELEASE);
396}
397
398/*
399 * pcu_save_all_on_cpu: save all PCU states on the current CPU.
400 */
401void
402pcu_save_all_on_cpu(void)
403{
404 int s;
405
406 s = splpcu();
407 for (u_int id = 0; id < PCU_UNIT_COUNT; id++) {
408 const pcu_ops_t * const pcu = pcu_ops_md_defs[id];
409 lwp_t *l;
410
411 if ((l = curcpu()->ci_pcu_curlwp[id]) != NULL) {
412 pcu_do_op(pcu, l, PCU_CMD_SAVE | PCU_CMD_RELEASE);
413 }
414 }
415 splx(s);
416}
417
418/*
419 * pcu_valid_p: return true if PCU state is considered valid. Generally,
420 * it always becomes "valid" when pcu_load() is called.
421 */
422bool
423pcu_valid_p(const pcu_ops_t *pcu)
424{
425 const u_int id = pcu->pcu_id;
426 lwp_t * const l = curlwp;
427
428 return (l->l_pcu_valid & (1U << id)) != 0;
429}
430
431#endif /* PCU_UNIT_COUNT > 0 */
432