1/* $NetBSD: subr_ipi.c,v 1.3 2015/01/18 23:16:35 rmind Exp $ */
2
3/*-
4 * Copyright (c) 2014 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Mindaugas Rasiukevicius.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32/*
33 * Inter-processor interrupt (IPI) interface: asynchronous IPIs to
34 * invoke functions with a constant argument and synchronous IPIs
35 * with the cross-call support.
36 */
37
38#include <sys/cdefs.h>
39__KERNEL_RCSID(0, "$NetBSD: subr_ipi.c,v 1.3 2015/01/18 23:16:35 rmind Exp $");
40
41#include <sys/param.h>
42#include <sys/types.h>
43
44#include <sys/atomic.h>
45#include <sys/evcnt.h>
46#include <sys/cpu.h>
47#include <sys/ipi.h>
48#include <sys/intr.h>
49#include <sys/kcpuset.h>
50#include <sys/kmem.h>
51#include <sys/lock.h>
52#include <sys/mutex.h>
53
54/*
55 * An array of the IPI handlers used for asynchronous invocation.
56 * The lock protects the slot allocation.
57 */
58
59typedef struct {
60 ipi_func_t func;
61 void * arg;
62} ipi_intr_t;
63
64static kmutex_t ipi_mngmt_lock;
65static ipi_intr_t ipi_intrs[IPI_MAXREG] __cacheline_aligned;
66
67/*
68 * Per-CPU mailbox for IPI messages: it is a single cache line storing
69 * up to IPI_MSG_MAX messages. This interface is built on top of the
70 * synchronous IPIs.
71 */
72
73#define IPI_MSG_SLOTS (CACHE_LINE_SIZE / sizeof(ipi_msg_t *))
74#define IPI_MSG_MAX IPI_MSG_SLOTS
75
76typedef struct {
77 ipi_msg_t * msg[IPI_MSG_SLOTS];
78} ipi_mbox_t;
79
80
81/* Mailboxes for the synchronous IPIs. */
82static ipi_mbox_t * ipi_mboxes __read_mostly;
83static struct evcnt ipi_mboxfull_ev __cacheline_aligned;
84static void ipi_msg_cpu_handler(void *);
85
86/* Handler for the synchronous IPIs - it must be zero. */
87#define IPI_SYNCH_ID 0
88
89#ifndef MULTIPROCESSOR
90#define cpu_ipi(ci) KASSERT(ci == NULL)
91#endif
92
93void
94ipi_sysinit(void)
95{
96 const size_t len = ncpu * sizeof(ipi_mbox_t);
97
98 /* Initialise the per-CPU bit fields. */
99 for (u_int i = 0; i < ncpu; i++) {
100 struct cpu_info *ci = cpu_lookup(i);
101 memset(&ci->ci_ipipend, 0, sizeof(ci->ci_ipipend));
102 }
103 mutex_init(&ipi_mngmt_lock, MUTEX_DEFAULT, IPL_NONE);
104 memset(ipi_intrs, 0, sizeof(ipi_intrs));
105
106 /* Allocate per-CPU IPI mailboxes. */
107 ipi_mboxes = kmem_zalloc(len, KM_SLEEP);
108 KASSERT(ipi_mboxes != NULL);
109
110 /*
111 * Register the handler for synchronous IPIs. This mechanism
112 * is built on top of the asynchronous interface. Slot zero is
113 * reserved permanently; it is also handy to use zero as a failure
114 * for other registers (as it is potentially less error-prone).
115 */
116 ipi_intrs[IPI_SYNCH_ID].func = ipi_msg_cpu_handler;
117
118 evcnt_attach_dynamic(&ipi_mboxfull_ev, EVCNT_TYPE_MISC, NULL,
119 "ipi", "full");
120}
121
122/*
123 * ipi_register: register an asynchronous IPI handler.
124 *
125 * => Returns IPI ID which is greater than zero; on failure - zero.
126 */
127u_int
128ipi_register(ipi_func_t func, void *arg)
129{
130 mutex_enter(&ipi_mngmt_lock);
131 for (u_int i = 0; i < IPI_MAXREG; i++) {
132 if (ipi_intrs[i].func == NULL) {
133 /* Register the function. */
134 ipi_intrs[i].func = func;
135 ipi_intrs[i].arg = arg;
136 mutex_exit(&ipi_mngmt_lock);
137
138 KASSERT(i != IPI_SYNCH_ID);
139 return i;
140 }
141 }
142 mutex_exit(&ipi_mngmt_lock);
143 printf("WARNING: ipi_register: table full, increase IPI_MAXREG\n");
144 return 0;
145}
146
147/*
148 * ipi_unregister: release the IPI handler given the ID.
149 */
150void
151ipi_unregister(u_int ipi_id)
152{
153 ipi_msg_t ipimsg = { .func = (ipi_func_t)nullop };
154
155 KASSERT(ipi_id != IPI_SYNCH_ID);
156 KASSERT(ipi_id < IPI_MAXREG);
157
158 /* Release the slot. */
159 mutex_enter(&ipi_mngmt_lock);
160 KASSERT(ipi_intrs[ipi_id].func != NULL);
161 ipi_intrs[ipi_id].func = NULL;
162
163 /* Ensure that there are no IPIs in flight. */
164 kpreempt_disable();
165 ipi_broadcast(&ipimsg);
166 ipi_wait(&ipimsg);
167 kpreempt_enable();
168 mutex_exit(&ipi_mngmt_lock);
169}
170
171/*
172 * ipi_trigger: asynchronously send an IPI to the specified CPU.
173 */
174void
175ipi_trigger(u_int ipi_id, struct cpu_info *ci)
176{
177 const u_int i = ipi_id >> IPI_BITW_SHIFT;
178 const uint32_t bitm = 1U << (ipi_id & IPI_BITW_MASK);
179
180 KASSERT(ipi_id < IPI_MAXREG);
181 KASSERT(kpreempt_disabled());
182 KASSERT(curcpu() != ci);
183
184 /* Mark as pending and send an IPI. */
185 if (membar_consumer(), (ci->ci_ipipend[i] & bitm) == 0) {
186 atomic_or_32(&ci->ci_ipipend[i], bitm);
187 cpu_ipi(ci);
188 }
189}
190
191/*
192 * ipi_trigger_multi: same as ipi_trigger() but sends to the multiple
193 * CPUs given the target CPU set.
194 */
195void
196ipi_trigger_multi(u_int ipi_id, const kcpuset_t *target)
197{
198 const cpuid_t selfid = cpu_index(curcpu());
199 CPU_INFO_ITERATOR cii;
200 struct cpu_info *ci;
201
202 KASSERT(kpreempt_disabled());
203 KASSERT(target != NULL);
204
205 for (CPU_INFO_FOREACH(cii, ci)) {
206 const cpuid_t cpuid = cpu_index(ci);
207
208 if (!kcpuset_isset(target, cpuid) || cpuid == selfid) {
209 continue;
210 }
211 ipi_trigger(ipi_id, ci);
212 }
213 if (kcpuset_isset(target, selfid)) {
214 int s = splhigh();
215 ipi_cpu_handler();
216 splx(s);
217 }
218}
219
220/*
221 * put_msg: insert message into the mailbox.
222 */
223static inline void
224put_msg(ipi_mbox_t *mbox, ipi_msg_t *msg)
225{
226 int count = SPINLOCK_BACKOFF_MIN;
227again:
228 for (u_int i = 0; i < IPI_MSG_MAX; i++) {
229 if (__predict_true(mbox->msg[i] == NULL) &&
230 atomic_cas_ptr(&mbox->msg[i], NULL, msg) == NULL) {
231 return;
232 }
233 }
234
235 /* All slots are full: we have to spin-wait. */
236 ipi_mboxfull_ev.ev_count++;
237 SPINLOCK_BACKOFF(count);
238 goto again;
239}
240
241/*
242 * ipi_cpu_handler: the IPI handler.
243 */
244void
245ipi_cpu_handler(void)
246{
247 struct cpu_info * const ci = curcpu();
248
249 /*
250 * Handle asynchronous IPIs: inspect per-CPU bit field, extract
251 * IPI ID numbers and execute functions in those slots.
252 */
253 for (u_int i = 0; i < IPI_BITWORDS; i++) {
254 uint32_t pending, bit;
255
256 if (ci->ci_ipipend[i] == 0) {
257 continue;
258 }
259 pending = atomic_swap_32(&ci->ci_ipipend[i], 0);
260#ifndef __HAVE_ATOMIC_AS_MEMBAR
261 membar_producer();
262#endif
263 while ((bit = ffs(pending)) != 0) {
264 const u_int ipi_id = (i << IPI_BITW_SHIFT) | --bit;
265 ipi_intr_t *ipi_hdl = &ipi_intrs[ipi_id];
266
267 pending &= ~(1U << bit);
268 KASSERT(ipi_hdl->func != NULL);
269 ipi_hdl->func(ipi_hdl->arg);
270 }
271 }
272}
273
274/*
275 * ipi_msg_cpu_handler: handle synchronous IPIs - iterate mailbox,
276 * execute the passed functions and acknowledge the messages.
277 */
278static void
279ipi_msg_cpu_handler(void *arg __unused)
280{
281 const struct cpu_info * const ci = curcpu();
282 ipi_mbox_t *mbox = &ipi_mboxes[cpu_index(ci)];
283
284 for (u_int i = 0; i < IPI_MSG_MAX; i++) {
285 ipi_msg_t *msg;
286
287 /* Get the message. */
288 if ((msg = mbox->msg[i]) == NULL) {
289 continue;
290 }
291 mbox->msg[i] = NULL;
292
293 /* Execute the handler. */
294 KASSERT(msg->func);
295 msg->func(msg->arg);
296
297 /* Ack the request. */
298 atomic_dec_uint(&msg->_pending);
299 }
300}
301
302/*
303 * ipi_unicast: send an IPI to a single CPU.
304 *
305 * => The CPU must be remote; must not be local.
306 * => The caller must ipi_wait() on the message for completion.
307 */
308void
309ipi_unicast(ipi_msg_t *msg, struct cpu_info *ci)
310{
311 const cpuid_t id = cpu_index(ci);
312
313 KASSERT(msg->func != NULL);
314 KASSERT(kpreempt_disabled());
315 KASSERT(curcpu() != ci);
316
317 msg->_pending = 1;
318 membar_producer();
319
320 put_msg(&ipi_mboxes[id], msg);
321 ipi_trigger(IPI_SYNCH_ID, ci);
322}
323
324/*
325 * ipi_multicast: send an IPI to each CPU in the specified set.
326 *
327 * => The caller must ipi_wait() on the message for completion.
328 */
329void
330ipi_multicast(ipi_msg_t *msg, const kcpuset_t *target)
331{
332 const struct cpu_info * const self = curcpu();
333 CPU_INFO_ITERATOR cii;
334 struct cpu_info *ci;
335 u_int local;
336
337 KASSERT(msg->func != NULL);
338 KASSERT(kpreempt_disabled());
339
340 local = !!kcpuset_isset(target, cpu_index(self));
341 msg->_pending = kcpuset_countset(target) - local;
342 membar_producer();
343
344 for (CPU_INFO_FOREACH(cii, ci)) {
345 cpuid_t id;
346
347 if (__predict_false(ci == self)) {
348 continue;
349 }
350 id = cpu_index(ci);
351 if (!kcpuset_isset(target, id)) {
352 continue;
353 }
354 put_msg(&ipi_mboxes[id], msg);
355 ipi_trigger(IPI_SYNCH_ID, ci);
356 }
357 if (local) {
358 msg->func(msg->arg);
359 }
360}
361
362/*
363 * ipi_broadcast: send an IPI to all CPUs.
364 *
365 * => The caller must ipi_wait() on the message for completion.
366 */
367void
368ipi_broadcast(ipi_msg_t *msg)
369{
370 const struct cpu_info * const self = curcpu();
371 CPU_INFO_ITERATOR cii;
372 struct cpu_info *ci;
373
374 KASSERT(msg->func != NULL);
375 KASSERT(kpreempt_disabled());
376
377 msg->_pending = ncpu - 1;
378 membar_producer();
379
380 /* Broadcast IPIs for remote CPUs. */
381 for (CPU_INFO_FOREACH(cii, ci)) {
382 cpuid_t id;
383
384 if (__predict_false(ci == self)) {
385 continue;
386 }
387 id = cpu_index(ci);
388 put_msg(&ipi_mboxes[id], msg);
389 ipi_trigger(IPI_SYNCH_ID, ci);
390 }
391
392 /* Finally, execute locally. */
393 msg->func(msg->arg);
394}
395
396/*
397 * ipi_wait: spin-wait until the message is processed.
398 */
399void
400ipi_wait(ipi_msg_t *msg)
401{
402 int count = SPINLOCK_BACKOFF_MIN;
403
404 while (msg->_pending) {
405 KASSERT(msg->_pending < ncpu);
406 SPINLOCK_BACKOFF(count);
407 }
408}
409