1 | /* $NetBSD: subr_ipi.c,v 1.3 2015/01/18 23:16:35 rmind Exp $ */ |
2 | |
3 | /*- |
4 | * Copyright (c) 2014 The NetBSD Foundation, Inc. |
5 | * All rights reserved. |
6 | * |
7 | * This code is derived from software contributed to The NetBSD Foundation |
8 | * by Mindaugas Rasiukevicius. |
9 | * |
10 | * Redistribution and use in source and binary forms, with or without |
11 | * modification, are permitted provided that the following conditions |
12 | * are met: |
13 | * 1. Redistributions of source code must retain the above copyright |
14 | * notice, this list of conditions and the following disclaimer. |
15 | * 2. Redistributions in binary form must reproduce the above copyright |
16 | * notice, this list of conditions and the following disclaimer in the |
17 | * documentation and/or other materials provided with the distribution. |
18 | * |
19 | * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS |
20 | * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED |
21 | * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
22 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS |
23 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
24 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
25 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS |
26 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN |
27 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
28 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
29 | * POSSIBILITY OF SUCH DAMAGE. |
30 | */ |
31 | |
32 | /* |
33 | * Inter-processor interrupt (IPI) interface: asynchronous IPIs to |
34 | * invoke functions with a constant argument and synchronous IPIs |
35 | * with the cross-call support. |
36 | */ |
37 | |
38 | #include <sys/cdefs.h> |
39 | __KERNEL_RCSID(0, "$NetBSD: subr_ipi.c,v 1.3 2015/01/18 23:16:35 rmind Exp $" ); |
40 | |
41 | #include <sys/param.h> |
42 | #include <sys/types.h> |
43 | |
44 | #include <sys/atomic.h> |
45 | #include <sys/evcnt.h> |
46 | #include <sys/cpu.h> |
47 | #include <sys/ipi.h> |
48 | #include <sys/intr.h> |
49 | #include <sys/kcpuset.h> |
50 | #include <sys/kmem.h> |
51 | #include <sys/lock.h> |
52 | #include <sys/mutex.h> |
53 | |
54 | /* |
55 | * An array of the IPI handlers used for asynchronous invocation. |
56 | * The lock protects the slot allocation. |
57 | */ |
58 | |
59 | typedef struct { |
60 | ipi_func_t func; |
61 | void * arg; |
62 | } ipi_intr_t; |
63 | |
64 | static kmutex_t ipi_mngmt_lock; |
65 | static ipi_intr_t ipi_intrs[IPI_MAXREG] __cacheline_aligned; |
66 | |
67 | /* |
68 | * Per-CPU mailbox for IPI messages: it is a single cache line storing |
69 | * up to IPI_MSG_MAX messages. This interface is built on top of the |
70 | * synchronous IPIs. |
71 | */ |
72 | |
73 | #define IPI_MSG_SLOTS (CACHE_LINE_SIZE / sizeof(ipi_msg_t *)) |
74 | #define IPI_MSG_MAX IPI_MSG_SLOTS |
75 | |
76 | typedef struct { |
77 | ipi_msg_t * msg[IPI_MSG_SLOTS]; |
78 | } ipi_mbox_t; |
79 | |
80 | |
81 | /* Mailboxes for the synchronous IPIs. */ |
82 | static ipi_mbox_t * ipi_mboxes __read_mostly; |
83 | static struct evcnt ipi_mboxfull_ev __cacheline_aligned; |
84 | static void ipi_msg_cpu_handler(void *); |
85 | |
86 | /* Handler for the synchronous IPIs - it must be zero. */ |
87 | #define IPI_SYNCH_ID 0 |
88 | |
89 | #ifndef MULTIPROCESSOR |
90 | #define cpu_ipi(ci) KASSERT(ci == NULL) |
91 | #endif |
92 | |
93 | void |
94 | ipi_sysinit(void) |
95 | { |
96 | const size_t len = ncpu * sizeof(ipi_mbox_t); |
97 | |
98 | /* Initialise the per-CPU bit fields. */ |
99 | for (u_int i = 0; i < ncpu; i++) { |
100 | struct cpu_info *ci = cpu_lookup(i); |
101 | memset(&ci->ci_ipipend, 0, sizeof(ci->ci_ipipend)); |
102 | } |
103 | mutex_init(&ipi_mngmt_lock, MUTEX_DEFAULT, IPL_NONE); |
104 | memset(ipi_intrs, 0, sizeof(ipi_intrs)); |
105 | |
106 | /* Allocate per-CPU IPI mailboxes. */ |
107 | ipi_mboxes = kmem_zalloc(len, KM_SLEEP); |
108 | KASSERT(ipi_mboxes != NULL); |
109 | |
110 | /* |
111 | * Register the handler for synchronous IPIs. This mechanism |
112 | * is built on top of the asynchronous interface. Slot zero is |
113 | * reserved permanently; it is also handy to use zero as a failure |
114 | * for other registers (as it is potentially less error-prone). |
115 | */ |
116 | ipi_intrs[IPI_SYNCH_ID].func = ipi_msg_cpu_handler; |
117 | |
118 | evcnt_attach_dynamic(&ipi_mboxfull_ev, EVCNT_TYPE_MISC, NULL, |
119 | "ipi" , "full" ); |
120 | } |
121 | |
122 | /* |
123 | * ipi_register: register an asynchronous IPI handler. |
124 | * |
125 | * => Returns IPI ID which is greater than zero; on failure - zero. |
126 | */ |
127 | u_int |
128 | ipi_register(ipi_func_t func, void *arg) |
129 | { |
130 | mutex_enter(&ipi_mngmt_lock); |
131 | for (u_int i = 0; i < IPI_MAXREG; i++) { |
132 | if (ipi_intrs[i].func == NULL) { |
133 | /* Register the function. */ |
134 | ipi_intrs[i].func = func; |
135 | ipi_intrs[i].arg = arg; |
136 | mutex_exit(&ipi_mngmt_lock); |
137 | |
138 | KASSERT(i != IPI_SYNCH_ID); |
139 | return i; |
140 | } |
141 | } |
142 | mutex_exit(&ipi_mngmt_lock); |
143 | printf("WARNING: ipi_register: table full, increase IPI_MAXREG\n" ); |
144 | return 0; |
145 | } |
146 | |
147 | /* |
148 | * ipi_unregister: release the IPI handler given the ID. |
149 | */ |
150 | void |
151 | ipi_unregister(u_int ipi_id) |
152 | { |
153 | ipi_msg_t ipimsg = { .func = (ipi_func_t)nullop }; |
154 | |
155 | KASSERT(ipi_id != IPI_SYNCH_ID); |
156 | KASSERT(ipi_id < IPI_MAXREG); |
157 | |
158 | /* Release the slot. */ |
159 | mutex_enter(&ipi_mngmt_lock); |
160 | KASSERT(ipi_intrs[ipi_id].func != NULL); |
161 | ipi_intrs[ipi_id].func = NULL; |
162 | |
163 | /* Ensure that there are no IPIs in flight. */ |
164 | kpreempt_disable(); |
165 | ipi_broadcast(&ipimsg); |
166 | ipi_wait(&ipimsg); |
167 | kpreempt_enable(); |
168 | mutex_exit(&ipi_mngmt_lock); |
169 | } |
170 | |
171 | /* |
172 | * ipi_trigger: asynchronously send an IPI to the specified CPU. |
173 | */ |
174 | void |
175 | ipi_trigger(u_int ipi_id, struct cpu_info *ci) |
176 | { |
177 | const u_int i = ipi_id >> IPI_BITW_SHIFT; |
178 | const uint32_t bitm = 1U << (ipi_id & IPI_BITW_MASK); |
179 | |
180 | KASSERT(ipi_id < IPI_MAXREG); |
181 | KASSERT(kpreempt_disabled()); |
182 | KASSERT(curcpu() != ci); |
183 | |
184 | /* Mark as pending and send an IPI. */ |
185 | if (membar_consumer(), (ci->ci_ipipend[i] & bitm) == 0) { |
186 | atomic_or_32(&ci->ci_ipipend[i], bitm); |
187 | cpu_ipi(ci); |
188 | } |
189 | } |
190 | |
191 | /* |
192 | * ipi_trigger_multi: same as ipi_trigger() but sends to the multiple |
193 | * CPUs given the target CPU set. |
194 | */ |
195 | void |
196 | ipi_trigger_multi(u_int ipi_id, const kcpuset_t *target) |
197 | { |
198 | const cpuid_t selfid = cpu_index(curcpu()); |
199 | CPU_INFO_ITERATOR cii; |
200 | struct cpu_info *ci; |
201 | |
202 | KASSERT(kpreempt_disabled()); |
203 | KASSERT(target != NULL); |
204 | |
205 | for (CPU_INFO_FOREACH(cii, ci)) { |
206 | const cpuid_t cpuid = cpu_index(ci); |
207 | |
208 | if (!kcpuset_isset(target, cpuid) || cpuid == selfid) { |
209 | continue; |
210 | } |
211 | ipi_trigger(ipi_id, ci); |
212 | } |
213 | if (kcpuset_isset(target, selfid)) { |
214 | int s = splhigh(); |
215 | ipi_cpu_handler(); |
216 | splx(s); |
217 | } |
218 | } |
219 | |
220 | /* |
221 | * put_msg: insert message into the mailbox. |
222 | */ |
223 | static inline void |
224 | put_msg(ipi_mbox_t *mbox, ipi_msg_t *msg) |
225 | { |
226 | int count = SPINLOCK_BACKOFF_MIN; |
227 | again: |
228 | for (u_int i = 0; i < IPI_MSG_MAX; i++) { |
229 | if (__predict_true(mbox->msg[i] == NULL) && |
230 | atomic_cas_ptr(&mbox->msg[i], NULL, msg) == NULL) { |
231 | return; |
232 | } |
233 | } |
234 | |
235 | /* All slots are full: we have to spin-wait. */ |
236 | ipi_mboxfull_ev.ev_count++; |
237 | SPINLOCK_BACKOFF(count); |
238 | goto again; |
239 | } |
240 | |
241 | /* |
242 | * ipi_cpu_handler: the IPI handler. |
243 | */ |
244 | void |
245 | ipi_cpu_handler(void) |
246 | { |
247 | struct cpu_info * const ci = curcpu(); |
248 | |
249 | /* |
250 | * Handle asynchronous IPIs: inspect per-CPU bit field, extract |
251 | * IPI ID numbers and execute functions in those slots. |
252 | */ |
253 | for (u_int i = 0; i < IPI_BITWORDS; i++) { |
254 | uint32_t pending, bit; |
255 | |
256 | if (ci->ci_ipipend[i] == 0) { |
257 | continue; |
258 | } |
259 | pending = atomic_swap_32(&ci->ci_ipipend[i], 0); |
260 | #ifndef __HAVE_ATOMIC_AS_MEMBAR |
261 | membar_producer(); |
262 | #endif |
263 | while ((bit = ffs(pending)) != 0) { |
264 | const u_int ipi_id = (i << IPI_BITW_SHIFT) | --bit; |
265 | ipi_intr_t *ipi_hdl = &ipi_intrs[ipi_id]; |
266 | |
267 | pending &= ~(1U << bit); |
268 | KASSERT(ipi_hdl->func != NULL); |
269 | ipi_hdl->func(ipi_hdl->arg); |
270 | } |
271 | } |
272 | } |
273 | |
274 | /* |
275 | * ipi_msg_cpu_handler: handle synchronous IPIs - iterate mailbox, |
276 | * execute the passed functions and acknowledge the messages. |
277 | */ |
278 | static void |
279 | ipi_msg_cpu_handler(void *arg __unused) |
280 | { |
281 | const struct cpu_info * const ci = curcpu(); |
282 | ipi_mbox_t *mbox = &ipi_mboxes[cpu_index(ci)]; |
283 | |
284 | for (u_int i = 0; i < IPI_MSG_MAX; i++) { |
285 | ipi_msg_t *msg; |
286 | |
287 | /* Get the message. */ |
288 | if ((msg = mbox->msg[i]) == NULL) { |
289 | continue; |
290 | } |
291 | mbox->msg[i] = NULL; |
292 | |
293 | /* Execute the handler. */ |
294 | KASSERT(msg->func); |
295 | msg->func(msg->arg); |
296 | |
297 | /* Ack the request. */ |
298 | atomic_dec_uint(&msg->_pending); |
299 | } |
300 | } |
301 | |
302 | /* |
303 | * ipi_unicast: send an IPI to a single CPU. |
304 | * |
305 | * => The CPU must be remote; must not be local. |
306 | * => The caller must ipi_wait() on the message for completion. |
307 | */ |
308 | void |
309 | ipi_unicast(ipi_msg_t *msg, struct cpu_info *ci) |
310 | { |
311 | const cpuid_t id = cpu_index(ci); |
312 | |
313 | KASSERT(msg->func != NULL); |
314 | KASSERT(kpreempt_disabled()); |
315 | KASSERT(curcpu() != ci); |
316 | |
317 | msg->_pending = 1; |
318 | membar_producer(); |
319 | |
320 | put_msg(&ipi_mboxes[id], msg); |
321 | ipi_trigger(IPI_SYNCH_ID, ci); |
322 | } |
323 | |
324 | /* |
325 | * ipi_multicast: send an IPI to each CPU in the specified set. |
326 | * |
327 | * => The caller must ipi_wait() on the message for completion. |
328 | */ |
329 | void |
330 | ipi_multicast(ipi_msg_t *msg, const kcpuset_t *target) |
331 | { |
332 | const struct cpu_info * const self = curcpu(); |
333 | CPU_INFO_ITERATOR cii; |
334 | struct cpu_info *ci; |
335 | u_int local; |
336 | |
337 | KASSERT(msg->func != NULL); |
338 | KASSERT(kpreempt_disabled()); |
339 | |
340 | local = !!kcpuset_isset(target, cpu_index(self)); |
341 | msg->_pending = kcpuset_countset(target) - local; |
342 | membar_producer(); |
343 | |
344 | for (CPU_INFO_FOREACH(cii, ci)) { |
345 | cpuid_t id; |
346 | |
347 | if (__predict_false(ci == self)) { |
348 | continue; |
349 | } |
350 | id = cpu_index(ci); |
351 | if (!kcpuset_isset(target, id)) { |
352 | continue; |
353 | } |
354 | put_msg(&ipi_mboxes[id], msg); |
355 | ipi_trigger(IPI_SYNCH_ID, ci); |
356 | } |
357 | if (local) { |
358 | msg->func(msg->arg); |
359 | } |
360 | } |
361 | |
362 | /* |
363 | * ipi_broadcast: send an IPI to all CPUs. |
364 | * |
365 | * => The caller must ipi_wait() on the message for completion. |
366 | */ |
367 | void |
368 | ipi_broadcast(ipi_msg_t *msg) |
369 | { |
370 | const struct cpu_info * const self = curcpu(); |
371 | CPU_INFO_ITERATOR cii; |
372 | struct cpu_info *ci; |
373 | |
374 | KASSERT(msg->func != NULL); |
375 | KASSERT(kpreempt_disabled()); |
376 | |
377 | msg->_pending = ncpu - 1; |
378 | membar_producer(); |
379 | |
380 | /* Broadcast IPIs for remote CPUs. */ |
381 | for (CPU_INFO_FOREACH(cii, ci)) { |
382 | cpuid_t id; |
383 | |
384 | if (__predict_false(ci == self)) { |
385 | continue; |
386 | } |
387 | id = cpu_index(ci); |
388 | put_msg(&ipi_mboxes[id], msg); |
389 | ipi_trigger(IPI_SYNCH_ID, ci); |
390 | } |
391 | |
392 | /* Finally, execute locally. */ |
393 | msg->func(msg->arg); |
394 | } |
395 | |
396 | /* |
397 | * ipi_wait: spin-wait until the message is processed. |
398 | */ |
399 | void |
400 | ipi_wait(ipi_msg_t *msg) |
401 | { |
402 | int count = SPINLOCK_BACKOFF_MIN; |
403 | |
404 | while (msg->_pending) { |
405 | KASSERT(msg->_pending < ncpu); |
406 | SPINLOCK_BACKOFF(count); |
407 | } |
408 | } |
409 | |