1 | /* $NetBSD: subr_xcall.c,v 1.19 2016/11/21 00:54:21 ozaki-r Exp $ */ |
2 | |
3 | /*- |
4 | * Copyright (c) 2007-2010 The NetBSD Foundation, Inc. |
5 | * All rights reserved. |
6 | * |
7 | * This code is derived from software contributed to The NetBSD Foundation |
8 | * by Andrew Doran and Mindaugas Rasiukevicius. |
9 | * |
10 | * Redistribution and use in source and binary forms, with or without |
11 | * modification, are permitted provided that the following conditions |
12 | * are met: |
13 | * 1. Redistributions of source code must retain the above copyright |
14 | * notice, this list of conditions and the following disclaimer. |
15 | * 2. Redistributions in binary form must reproduce the above copyright |
16 | * notice, this list of conditions and the following disclaimer in the |
17 | * documentation and/or other materials provided with the distribution. |
18 | * |
19 | * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS |
20 | * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED |
21 | * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
22 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS |
23 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
24 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
25 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS |
26 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN |
27 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
28 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
29 | * POSSIBILITY OF SUCH DAMAGE. |
30 | */ |
31 | |
32 | /* |
33 | * Cross call support |
34 | * |
35 | * Background |
36 | * |
37 | * Sometimes it is necessary to modify hardware state that is tied |
38 | * directly to individual CPUs (such as a CPU's local timer), and |
39 | * these updates can not be done remotely by another CPU. The LWP |
40 | * requesting the update may be unable to guarantee that it will be |
41 | * running on the CPU where the update must occur, when the update |
42 | * occurs. |
43 | * |
44 | * Additionally, it's sometimes necessary to modify per-CPU software |
45 | * state from a remote CPU. Where these update operations are so |
46 | * rare or the access to the per-CPU data so frequent that the cost |
47 | * of using locking or atomic operations to provide coherency is |
48 | * prohibitive, another way must be found. |
49 | * |
50 | * Cross calls help to solve these types of problem by allowing |
51 | * any CPU in the system to request that an arbitrary function be |
52 | * executed on any other CPU. |
53 | * |
54 | * Implementation |
55 | * |
56 | * A slow mechanism for making 'low priority' cross calls is |
57 | * provided. The function to be executed runs on the remote CPU |
58 | * within a bound kthread. No queueing is provided, and the |
59 | * implementation uses global state. The function being called may |
60 | * block briefly on locks, but in doing so must be careful to not |
61 | * interfere with other cross calls in the system. The function is |
62 | * called with thread context and not from a soft interrupt, so it |
63 | * can ensure that it is not interrupting other code running on the |
64 | * CPU, and so has exclusive access to the CPU. Since this facility |
65 | * is heavyweight, it's expected that it will not be used often. |
66 | * |
67 | * Cross calls must not allocate memory, as the pagedaemon uses |
68 | * them (and memory allocation may need to wait on the pagedaemon). |
69 | * |
70 | * A low-overhead mechanism for high priority calls (XC_HIGHPRI) is |
71 | * also provided. The function to be executed runs on a software |
72 | * interrupt context, at IPL_SOFTSERIAL level, and is expected to |
73 | * be very lightweight, e.g. avoid blocking. |
74 | */ |
75 | |
76 | #include <sys/cdefs.h> |
77 | __KERNEL_RCSID(0, "$NetBSD: subr_xcall.c,v 1.19 2016/11/21 00:54:21 ozaki-r Exp $" ); |
78 | |
79 | #include <sys/types.h> |
80 | #include <sys/param.h> |
81 | #include <sys/xcall.h> |
82 | #include <sys/mutex.h> |
83 | #include <sys/condvar.h> |
84 | #include <sys/evcnt.h> |
85 | #include <sys/kthread.h> |
86 | #include <sys/cpu.h> |
87 | |
88 | #ifdef _RUMPKERNEL |
89 | #include "rump_private.h" |
90 | #endif |
91 | |
92 | /* Cross-call state box. */ |
93 | typedef struct { |
94 | kmutex_t xc_lock; |
95 | kcondvar_t xc_busy; |
96 | xcfunc_t xc_func; |
97 | void * xc_arg1; |
98 | void * xc_arg2; |
99 | uint64_t xc_headp; |
100 | uint64_t xc_donep; |
101 | } xc_state_t; |
102 | |
103 | /* Bit indicating high (1) or low (0) priority. */ |
104 | #define XC_PRI_BIT (1ULL << 63) |
105 | |
106 | /* Low priority xcall structures. */ |
107 | static xc_state_t xc_low_pri __cacheline_aligned; |
108 | |
109 | /* High priority xcall structures. */ |
110 | static xc_state_t xc_high_pri __cacheline_aligned; |
111 | static void * xc_sih __cacheline_aligned; |
112 | |
113 | /* Event counters. */ |
114 | static struct evcnt xc_unicast_ev __cacheline_aligned; |
115 | static struct evcnt xc_broadcast_ev __cacheline_aligned; |
116 | |
117 | static void xc_init(void); |
118 | static void xc_thread(void *); |
119 | |
120 | static inline uint64_t xc_highpri(xcfunc_t, void *, void *, struct cpu_info *); |
121 | static inline uint64_t xc_lowpri(xcfunc_t, void *, void *, struct cpu_info *); |
122 | |
123 | /* |
124 | * xc_init: |
125 | * |
126 | * Initialize low and high priority cross-call structures. |
127 | */ |
128 | static void |
129 | xc_init(void) |
130 | { |
131 | xc_state_t *xclo = &xc_low_pri, *xchi = &xc_high_pri; |
132 | |
133 | memset(xclo, 0, sizeof(xc_state_t)); |
134 | mutex_init(&xclo->xc_lock, MUTEX_DEFAULT, IPL_NONE); |
135 | cv_init(&xclo->xc_busy, "xclocv" ); |
136 | |
137 | memset(xchi, 0, sizeof(xc_state_t)); |
138 | mutex_init(&xchi->xc_lock, MUTEX_DEFAULT, IPL_SOFTSERIAL); |
139 | cv_init(&xchi->xc_busy, "xchicv" ); |
140 | xc_sih = softint_establish(SOFTINT_SERIAL | SOFTINT_MPSAFE, |
141 | xc__highpri_intr, NULL); |
142 | KASSERT(xc_sih != NULL); |
143 | |
144 | evcnt_attach_dynamic(&xc_unicast_ev, EVCNT_TYPE_MISC, NULL, |
145 | "crosscall" , "unicast" ); |
146 | evcnt_attach_dynamic(&xc_broadcast_ev, EVCNT_TYPE_MISC, NULL, |
147 | "crosscall" , "broadcast" ); |
148 | } |
149 | |
150 | /* |
151 | * xc_init_cpu: |
152 | * |
153 | * Initialize the cross-call subsystem. Called once for each CPU |
154 | * in the system as they are attached. |
155 | */ |
156 | void |
157 | xc_init_cpu(struct cpu_info *ci) |
158 | { |
159 | static bool again = false; |
160 | int error __diagused; |
161 | |
162 | if (!again) { |
163 | /* Autoconfiguration will prevent re-entry. */ |
164 | xc_init(); |
165 | again = true; |
166 | } |
167 | cv_init(&ci->ci_data.cpu_xcall, "xcall" ); |
168 | error = kthread_create(PRI_XCALL, KTHREAD_MPSAFE, ci, xc_thread, |
169 | NULL, NULL, "xcall/%u" , ci->ci_index); |
170 | KASSERT(error == 0); |
171 | } |
172 | |
173 | /* |
174 | * xc_broadcast: |
175 | * |
176 | * Trigger a call on all CPUs in the system. |
177 | */ |
178 | uint64_t |
179 | xc_broadcast(u_int flags, xcfunc_t func, void *arg1, void *arg2) |
180 | { |
181 | |
182 | KASSERT(!cpu_intr_p() && !cpu_softintr_p()); |
183 | |
184 | if ((flags & XC_HIGHPRI) != 0) { |
185 | return xc_highpri(func, arg1, arg2, NULL); |
186 | } else { |
187 | return xc_lowpri(func, arg1, arg2, NULL); |
188 | } |
189 | } |
190 | |
191 | /* |
192 | * xc_unicast: |
193 | * |
194 | * Trigger a call on one CPU. |
195 | */ |
196 | uint64_t |
197 | xc_unicast(u_int flags, xcfunc_t func, void *arg1, void *arg2, |
198 | struct cpu_info *ci) |
199 | { |
200 | |
201 | KASSERT(ci != NULL); |
202 | KASSERT(!cpu_intr_p() && !cpu_softintr_p()); |
203 | |
204 | if ((flags & XC_HIGHPRI) != 0) { |
205 | return xc_highpri(func, arg1, arg2, ci); |
206 | } else { |
207 | return xc_lowpri(func, arg1, arg2, ci); |
208 | } |
209 | } |
210 | |
211 | /* |
212 | * xc_wait: |
213 | * |
214 | * Wait for a cross call to complete. |
215 | */ |
216 | void |
217 | xc_wait(uint64_t where) |
218 | { |
219 | xc_state_t *xc; |
220 | |
221 | KASSERT(!cpu_intr_p() && !cpu_softintr_p()); |
222 | |
223 | /* Determine whether it is high or low priority cross-call. */ |
224 | if ((where & XC_PRI_BIT) != 0) { |
225 | xc = &xc_high_pri; |
226 | where &= ~XC_PRI_BIT; |
227 | } else { |
228 | xc = &xc_low_pri; |
229 | } |
230 | |
231 | /* Fast path, if already done. */ |
232 | if (xc->xc_donep >= where) { |
233 | return; |
234 | } |
235 | |
236 | /* Slow path: block until awoken. */ |
237 | mutex_enter(&xc->xc_lock); |
238 | while (xc->xc_donep < where) { |
239 | cv_wait(&xc->xc_busy, &xc->xc_lock); |
240 | } |
241 | mutex_exit(&xc->xc_lock); |
242 | } |
243 | |
244 | /* |
245 | * xc_lowpri: |
246 | * |
247 | * Trigger a low priority call on one or more CPUs. |
248 | */ |
249 | static inline uint64_t |
250 | xc_lowpri(xcfunc_t func, void *arg1, void *arg2, struct cpu_info *ci) |
251 | { |
252 | xc_state_t *xc = &xc_low_pri; |
253 | CPU_INFO_ITERATOR cii; |
254 | uint64_t where; |
255 | |
256 | mutex_enter(&xc->xc_lock); |
257 | while (xc->xc_headp != xc->xc_donep) { |
258 | cv_wait(&xc->xc_busy, &xc->xc_lock); |
259 | } |
260 | xc->xc_arg1 = arg1; |
261 | xc->xc_arg2 = arg2; |
262 | xc->xc_func = func; |
263 | if (ci == NULL) { |
264 | xc_broadcast_ev.ev_count++; |
265 | for (CPU_INFO_FOREACH(cii, ci)) { |
266 | if ((ci->ci_schedstate.spc_flags & SPCF_RUNNING) == 0) |
267 | continue; |
268 | xc->xc_headp += 1; |
269 | ci->ci_data.cpu_xcall_pending = true; |
270 | cv_signal(&ci->ci_data.cpu_xcall); |
271 | } |
272 | } else { |
273 | xc_unicast_ev.ev_count++; |
274 | xc->xc_headp += 1; |
275 | ci->ci_data.cpu_xcall_pending = true; |
276 | cv_signal(&ci->ci_data.cpu_xcall); |
277 | } |
278 | KASSERT(xc->xc_donep < xc->xc_headp); |
279 | where = xc->xc_headp; |
280 | mutex_exit(&xc->xc_lock); |
281 | |
282 | /* Return a low priority ticket. */ |
283 | KASSERT((where & XC_PRI_BIT) == 0); |
284 | return where; |
285 | } |
286 | |
287 | /* |
288 | * xc_thread: |
289 | * |
290 | * One thread per-CPU to dispatch low priority calls. |
291 | */ |
292 | static void |
293 | xc_thread(void *cookie) |
294 | { |
295 | struct cpu_info *ci = curcpu(); |
296 | xc_state_t *xc = &xc_low_pri; |
297 | void *arg1, *arg2; |
298 | xcfunc_t func; |
299 | |
300 | mutex_enter(&xc->xc_lock); |
301 | for (;;) { |
302 | while (!ci->ci_data.cpu_xcall_pending) { |
303 | if (xc->xc_headp == xc->xc_donep) { |
304 | cv_broadcast(&xc->xc_busy); |
305 | } |
306 | cv_wait(&ci->ci_data.cpu_xcall, &xc->xc_lock); |
307 | KASSERT(ci == curcpu()); |
308 | } |
309 | ci->ci_data.cpu_xcall_pending = false; |
310 | func = xc->xc_func; |
311 | arg1 = xc->xc_arg1; |
312 | arg2 = xc->xc_arg2; |
313 | mutex_exit(&xc->xc_lock); |
314 | |
315 | KASSERT(func != NULL); |
316 | (*func)(arg1, arg2); |
317 | |
318 | mutex_enter(&xc->xc_lock); |
319 | xc->xc_donep++; |
320 | } |
321 | /* NOTREACHED */ |
322 | } |
323 | |
324 | /* |
325 | * xc_ipi_handler: |
326 | * |
327 | * Handler of cross-call IPI. |
328 | */ |
329 | void |
330 | xc_ipi_handler(void) |
331 | { |
332 | /* Executes xc__highpri_intr() via software interrupt. */ |
333 | softint_schedule(xc_sih); |
334 | } |
335 | |
336 | /* |
337 | * xc__highpri_intr: |
338 | * |
339 | * A software interrupt handler for high priority calls. |
340 | */ |
341 | void |
342 | xc__highpri_intr(void *dummy) |
343 | { |
344 | xc_state_t *xc = &xc_high_pri; |
345 | void *arg1, *arg2; |
346 | xcfunc_t func; |
347 | |
348 | KASSERT(!cpu_intr_p()); |
349 | /* |
350 | * Lock-less fetch of function and its arguments. |
351 | * Safe since it cannot change at this point. |
352 | */ |
353 | KASSERT(xc->xc_donep < xc->xc_headp); |
354 | func = xc->xc_func; |
355 | arg1 = xc->xc_arg1; |
356 | arg2 = xc->xc_arg2; |
357 | |
358 | KASSERT(func != NULL); |
359 | (*func)(arg1, arg2); |
360 | |
361 | /* |
362 | * Note the request as done, and if we have reached the head, |
363 | * cross-call has been processed - notify waiters, if any. |
364 | */ |
365 | mutex_enter(&xc->xc_lock); |
366 | if (++xc->xc_donep == xc->xc_headp) { |
367 | cv_broadcast(&xc->xc_busy); |
368 | } |
369 | mutex_exit(&xc->xc_lock); |
370 | } |
371 | |
372 | /* |
373 | * xc_highpri: |
374 | * |
375 | * Trigger a high priority call on one or more CPUs. |
376 | */ |
377 | static inline uint64_t |
378 | xc_highpri(xcfunc_t func, void *arg1, void *arg2, struct cpu_info *ci) |
379 | { |
380 | xc_state_t *xc = &xc_high_pri; |
381 | uint64_t where; |
382 | |
383 | mutex_enter(&xc->xc_lock); |
384 | while (xc->xc_headp != xc->xc_donep) { |
385 | cv_wait(&xc->xc_busy, &xc->xc_lock); |
386 | } |
387 | xc->xc_func = func; |
388 | xc->xc_arg1 = arg1; |
389 | xc->xc_arg2 = arg2; |
390 | xc->xc_headp += (ci ? 1 : ncpu); |
391 | where = xc->xc_headp; |
392 | mutex_exit(&xc->xc_lock); |
393 | |
394 | /* |
395 | * Send the IPI once lock is released. |
396 | * Note: it will handle the local CPU case. |
397 | */ |
398 | |
399 | #ifdef _RUMPKERNEL |
400 | rump_xc_highpri(ci); |
401 | #else |
402 | #ifdef MULTIPROCESSOR |
403 | kpreempt_disable(); |
404 | if (curcpu() == ci) { |
405 | /* Unicast: local CPU. */ |
406 | xc_ipi_handler(); |
407 | } else if (ci) { |
408 | /* Unicast: remote CPU. */ |
409 | xc_send_ipi(ci); |
410 | } else { |
411 | /* Broadcast: all, including local. */ |
412 | xc_send_ipi(NULL); |
413 | xc_ipi_handler(); |
414 | } |
415 | kpreempt_enable(); |
416 | #else |
417 | KASSERT(ci == NULL || curcpu() == ci); |
418 | xc_ipi_handler(); |
419 | #endif |
420 | #endif |
421 | |
422 | /* Indicate a high priority ticket. */ |
423 | return (where | XC_PRI_BIT); |
424 | } |
425 | |