1/* $NetBSD: subr_xcall.c,v 1.19 2016/11/21 00:54:21 ozaki-r Exp $ */
2
3/*-
4 * Copyright (c) 2007-2010 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Andrew Doran and Mindaugas Rasiukevicius.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32/*
33 * Cross call support
34 *
35 * Background
36 *
37 * Sometimes it is necessary to modify hardware state that is tied
38 * directly to individual CPUs (such as a CPU's local timer), and
39 * these updates can not be done remotely by another CPU. The LWP
40 * requesting the update may be unable to guarantee that it will be
41 * running on the CPU where the update must occur, when the update
42 * occurs.
43 *
44 * Additionally, it's sometimes necessary to modify per-CPU software
45 * state from a remote CPU. Where these update operations are so
46 * rare or the access to the per-CPU data so frequent that the cost
47 * of using locking or atomic operations to provide coherency is
48 * prohibitive, another way must be found.
49 *
50 * Cross calls help to solve these types of problem by allowing
51 * any CPU in the system to request that an arbitrary function be
52 * executed on any other CPU.
53 *
54 * Implementation
55 *
56 * A slow mechanism for making 'low priority' cross calls is
57 * provided. The function to be executed runs on the remote CPU
58 * within a bound kthread. No queueing is provided, and the
59 * implementation uses global state. The function being called may
60 * block briefly on locks, but in doing so must be careful to not
61 * interfere with other cross calls in the system. The function is
62 * called with thread context and not from a soft interrupt, so it
63 * can ensure that it is not interrupting other code running on the
64 * CPU, and so has exclusive access to the CPU. Since this facility
65 * is heavyweight, it's expected that it will not be used often.
66 *
67 * Cross calls must not allocate memory, as the pagedaemon uses
68 * them (and memory allocation may need to wait on the pagedaemon).
69 *
70 * A low-overhead mechanism for high priority calls (XC_HIGHPRI) is
71 * also provided. The function to be executed runs on a software
72 * interrupt context, at IPL_SOFTSERIAL level, and is expected to
73 * be very lightweight, e.g. avoid blocking.
74 */
75
76#include <sys/cdefs.h>
77__KERNEL_RCSID(0, "$NetBSD: subr_xcall.c,v 1.19 2016/11/21 00:54:21 ozaki-r Exp $");
78
79#include <sys/types.h>
80#include <sys/param.h>
81#include <sys/xcall.h>
82#include <sys/mutex.h>
83#include <sys/condvar.h>
84#include <sys/evcnt.h>
85#include <sys/kthread.h>
86#include <sys/cpu.h>
87
88#ifdef _RUMPKERNEL
89#include "rump_private.h"
90#endif
91
92/* Cross-call state box. */
93typedef struct {
94 kmutex_t xc_lock;
95 kcondvar_t xc_busy;
96 xcfunc_t xc_func;
97 void * xc_arg1;
98 void * xc_arg2;
99 uint64_t xc_headp;
100 uint64_t xc_donep;
101} xc_state_t;
102
103/* Bit indicating high (1) or low (0) priority. */
104#define XC_PRI_BIT (1ULL << 63)
105
106/* Low priority xcall structures. */
107static xc_state_t xc_low_pri __cacheline_aligned;
108
109/* High priority xcall structures. */
110static xc_state_t xc_high_pri __cacheline_aligned;
111static void * xc_sih __cacheline_aligned;
112
113/* Event counters. */
114static struct evcnt xc_unicast_ev __cacheline_aligned;
115static struct evcnt xc_broadcast_ev __cacheline_aligned;
116
117static void xc_init(void);
118static void xc_thread(void *);
119
120static inline uint64_t xc_highpri(xcfunc_t, void *, void *, struct cpu_info *);
121static inline uint64_t xc_lowpri(xcfunc_t, void *, void *, struct cpu_info *);
122
123/*
124 * xc_init:
125 *
126 * Initialize low and high priority cross-call structures.
127 */
128static void
129xc_init(void)
130{
131 xc_state_t *xclo = &xc_low_pri, *xchi = &xc_high_pri;
132
133 memset(xclo, 0, sizeof(xc_state_t));
134 mutex_init(&xclo->xc_lock, MUTEX_DEFAULT, IPL_NONE);
135 cv_init(&xclo->xc_busy, "xclocv");
136
137 memset(xchi, 0, sizeof(xc_state_t));
138 mutex_init(&xchi->xc_lock, MUTEX_DEFAULT, IPL_SOFTSERIAL);
139 cv_init(&xchi->xc_busy, "xchicv");
140 xc_sih = softint_establish(SOFTINT_SERIAL | SOFTINT_MPSAFE,
141 xc__highpri_intr, NULL);
142 KASSERT(xc_sih != NULL);
143
144 evcnt_attach_dynamic(&xc_unicast_ev, EVCNT_TYPE_MISC, NULL,
145 "crosscall", "unicast");
146 evcnt_attach_dynamic(&xc_broadcast_ev, EVCNT_TYPE_MISC, NULL,
147 "crosscall", "broadcast");
148}
149
150/*
151 * xc_init_cpu:
152 *
153 * Initialize the cross-call subsystem. Called once for each CPU
154 * in the system as they are attached.
155 */
156void
157xc_init_cpu(struct cpu_info *ci)
158{
159 static bool again = false;
160 int error __diagused;
161
162 if (!again) {
163 /* Autoconfiguration will prevent re-entry. */
164 xc_init();
165 again = true;
166 }
167 cv_init(&ci->ci_data.cpu_xcall, "xcall");
168 error = kthread_create(PRI_XCALL, KTHREAD_MPSAFE, ci, xc_thread,
169 NULL, NULL, "xcall/%u", ci->ci_index);
170 KASSERT(error == 0);
171}
172
173/*
174 * xc_broadcast:
175 *
176 * Trigger a call on all CPUs in the system.
177 */
178uint64_t
179xc_broadcast(u_int flags, xcfunc_t func, void *arg1, void *arg2)
180{
181
182 KASSERT(!cpu_intr_p() && !cpu_softintr_p());
183
184 if ((flags & XC_HIGHPRI) != 0) {
185 return xc_highpri(func, arg1, arg2, NULL);
186 } else {
187 return xc_lowpri(func, arg1, arg2, NULL);
188 }
189}
190
191/*
192 * xc_unicast:
193 *
194 * Trigger a call on one CPU.
195 */
196uint64_t
197xc_unicast(u_int flags, xcfunc_t func, void *arg1, void *arg2,
198 struct cpu_info *ci)
199{
200
201 KASSERT(ci != NULL);
202 KASSERT(!cpu_intr_p() && !cpu_softintr_p());
203
204 if ((flags & XC_HIGHPRI) != 0) {
205 return xc_highpri(func, arg1, arg2, ci);
206 } else {
207 return xc_lowpri(func, arg1, arg2, ci);
208 }
209}
210
211/*
212 * xc_wait:
213 *
214 * Wait for a cross call to complete.
215 */
216void
217xc_wait(uint64_t where)
218{
219 xc_state_t *xc;
220
221 KASSERT(!cpu_intr_p() && !cpu_softintr_p());
222
223 /* Determine whether it is high or low priority cross-call. */
224 if ((where & XC_PRI_BIT) != 0) {
225 xc = &xc_high_pri;
226 where &= ~XC_PRI_BIT;
227 } else {
228 xc = &xc_low_pri;
229 }
230
231 /* Fast path, if already done. */
232 if (xc->xc_donep >= where) {
233 return;
234 }
235
236 /* Slow path: block until awoken. */
237 mutex_enter(&xc->xc_lock);
238 while (xc->xc_donep < where) {
239 cv_wait(&xc->xc_busy, &xc->xc_lock);
240 }
241 mutex_exit(&xc->xc_lock);
242}
243
244/*
245 * xc_lowpri:
246 *
247 * Trigger a low priority call on one or more CPUs.
248 */
249static inline uint64_t
250xc_lowpri(xcfunc_t func, void *arg1, void *arg2, struct cpu_info *ci)
251{
252 xc_state_t *xc = &xc_low_pri;
253 CPU_INFO_ITERATOR cii;
254 uint64_t where;
255
256 mutex_enter(&xc->xc_lock);
257 while (xc->xc_headp != xc->xc_donep) {
258 cv_wait(&xc->xc_busy, &xc->xc_lock);
259 }
260 xc->xc_arg1 = arg1;
261 xc->xc_arg2 = arg2;
262 xc->xc_func = func;
263 if (ci == NULL) {
264 xc_broadcast_ev.ev_count++;
265 for (CPU_INFO_FOREACH(cii, ci)) {
266 if ((ci->ci_schedstate.spc_flags & SPCF_RUNNING) == 0)
267 continue;
268 xc->xc_headp += 1;
269 ci->ci_data.cpu_xcall_pending = true;
270 cv_signal(&ci->ci_data.cpu_xcall);
271 }
272 } else {
273 xc_unicast_ev.ev_count++;
274 xc->xc_headp += 1;
275 ci->ci_data.cpu_xcall_pending = true;
276 cv_signal(&ci->ci_data.cpu_xcall);
277 }
278 KASSERT(xc->xc_donep < xc->xc_headp);
279 where = xc->xc_headp;
280 mutex_exit(&xc->xc_lock);
281
282 /* Return a low priority ticket. */
283 KASSERT((where & XC_PRI_BIT) == 0);
284 return where;
285}
286
287/*
288 * xc_thread:
289 *
290 * One thread per-CPU to dispatch low priority calls.
291 */
292static void
293xc_thread(void *cookie)
294{
295 struct cpu_info *ci = curcpu();
296 xc_state_t *xc = &xc_low_pri;
297 void *arg1, *arg2;
298 xcfunc_t func;
299
300 mutex_enter(&xc->xc_lock);
301 for (;;) {
302 while (!ci->ci_data.cpu_xcall_pending) {
303 if (xc->xc_headp == xc->xc_donep) {
304 cv_broadcast(&xc->xc_busy);
305 }
306 cv_wait(&ci->ci_data.cpu_xcall, &xc->xc_lock);
307 KASSERT(ci == curcpu());
308 }
309 ci->ci_data.cpu_xcall_pending = false;
310 func = xc->xc_func;
311 arg1 = xc->xc_arg1;
312 arg2 = xc->xc_arg2;
313 mutex_exit(&xc->xc_lock);
314
315 KASSERT(func != NULL);
316 (*func)(arg1, arg2);
317
318 mutex_enter(&xc->xc_lock);
319 xc->xc_donep++;
320 }
321 /* NOTREACHED */
322}
323
324/*
325 * xc_ipi_handler:
326 *
327 * Handler of cross-call IPI.
328 */
329void
330xc_ipi_handler(void)
331{
332 /* Executes xc__highpri_intr() via software interrupt. */
333 softint_schedule(xc_sih);
334}
335
336/*
337 * xc__highpri_intr:
338 *
339 * A software interrupt handler for high priority calls.
340 */
341void
342xc__highpri_intr(void *dummy)
343{
344 xc_state_t *xc = &xc_high_pri;
345 void *arg1, *arg2;
346 xcfunc_t func;
347
348 KASSERT(!cpu_intr_p());
349 /*
350 * Lock-less fetch of function and its arguments.
351 * Safe since it cannot change at this point.
352 */
353 KASSERT(xc->xc_donep < xc->xc_headp);
354 func = xc->xc_func;
355 arg1 = xc->xc_arg1;
356 arg2 = xc->xc_arg2;
357
358 KASSERT(func != NULL);
359 (*func)(arg1, arg2);
360
361 /*
362 * Note the request as done, and if we have reached the head,
363 * cross-call has been processed - notify waiters, if any.
364 */
365 mutex_enter(&xc->xc_lock);
366 if (++xc->xc_donep == xc->xc_headp) {
367 cv_broadcast(&xc->xc_busy);
368 }
369 mutex_exit(&xc->xc_lock);
370}
371
372/*
373 * xc_highpri:
374 *
375 * Trigger a high priority call on one or more CPUs.
376 */
377static inline uint64_t
378xc_highpri(xcfunc_t func, void *arg1, void *arg2, struct cpu_info *ci)
379{
380 xc_state_t *xc = &xc_high_pri;
381 uint64_t where;
382
383 mutex_enter(&xc->xc_lock);
384 while (xc->xc_headp != xc->xc_donep) {
385 cv_wait(&xc->xc_busy, &xc->xc_lock);
386 }
387 xc->xc_func = func;
388 xc->xc_arg1 = arg1;
389 xc->xc_arg2 = arg2;
390 xc->xc_headp += (ci ? 1 : ncpu);
391 where = xc->xc_headp;
392 mutex_exit(&xc->xc_lock);
393
394 /*
395 * Send the IPI once lock is released.
396 * Note: it will handle the local CPU case.
397 */
398
399#ifdef _RUMPKERNEL
400 rump_xc_highpri(ci);
401#else
402#ifdef MULTIPROCESSOR
403 kpreempt_disable();
404 if (curcpu() == ci) {
405 /* Unicast: local CPU. */
406 xc_ipi_handler();
407 } else if (ci) {
408 /* Unicast: remote CPU. */
409 xc_send_ipi(ci);
410 } else {
411 /* Broadcast: all, including local. */
412 xc_send_ipi(NULL);
413 xc_ipi_handler();
414 }
415 kpreempt_enable();
416#else
417 KASSERT(ci == NULL || curcpu() == ci);
418 xc_ipi_handler();
419#endif
420#endif
421
422 /* Indicate a high priority ticket. */
423 return (where | XC_PRI_BIT);
424}
425