1/* $NetBSD: subr_interrupt.c,v 1.1 2015/08/17 06:16:03 knakahara Exp $ */
2
3/*
4 * Copyright (c) 2015 Internet Initiative Japan Inc.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
17 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
18 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26 * POSSIBILITY OF SUCH DAMAGE.
27 */
28
29#include <sys/cdefs.h>
30__KERNEL_RCSID(0, "$NetBSD: subr_interrupt.c,v 1.1 2015/08/17 06:16:03 knakahara Exp $");
31
32#include <sys/param.h>
33#include <sys/systm.h>
34#include <sys/kernel.h>
35#include <sys/errno.h>
36#include <sys/cpu.h>
37#include <sys/interrupt.h>
38#include <sys/intr.h>
39#include <sys/kcpuset.h>
40#include <sys/kmem.h>
41#include <sys/proc.h>
42#include <sys/xcall.h>
43#include <sys/sysctl.h>
44
45#include <sys/conf.h>
46#include <sys/intrio.h>
47#include <sys/kauth.h>
48
49#include <machine/limits.h>
50
51#ifdef INTR_DEBUG
52#define DPRINTF(msg) printf msg
53#else
54#define DPRINTF(msg)
55#endif
56
57static struct intrio_set kintrio_set = { "\0", NULL, 0 };
58
59#define UNSET_NOINTR_SHIELD 0
60#define SET_NOINTR_SHIELD 1
61
62static void
63interrupt_shield_xcall(void *arg1, void *arg2)
64{
65 struct cpu_info *ci;
66 struct schedstate_percpu *spc;
67 int s, shield;
68
69 ci = arg1;
70 shield = (int)(intptr_t)arg2;
71 spc = &ci->ci_schedstate;
72
73 s = splsched();
74 if (shield == UNSET_NOINTR_SHIELD)
75 spc->spc_flags &= ~SPCF_NOINTR;
76 else if (shield == SET_NOINTR_SHIELD)
77 spc->spc_flags |= SPCF_NOINTR;
78 splx(s);
79}
80
81/*
82 * Change SPCF_NOINTR flag of schedstate_percpu->spc_flags.
83 */
84static int
85interrupt_shield(u_int cpu_idx, int shield)
86{
87 struct cpu_info *ci;
88 struct schedstate_percpu *spc;
89
90 KASSERT(mutex_owned(&cpu_lock));
91
92 ci = cpu_lookup(cpu_idx);
93 if (ci == NULL)
94 return EINVAL;
95
96 spc = &ci->ci_schedstate;
97 if (shield == UNSET_NOINTR_SHIELD) {
98 if ((spc->spc_flags & SPCF_NOINTR) == 0)
99 return 0;
100 } else if (shield == SET_NOINTR_SHIELD) {
101 if ((spc->spc_flags & SPCF_NOINTR) != 0)
102 return 0;
103 }
104
105 if (ci == curcpu() || !mp_online) {
106 interrupt_shield_xcall(ci, (void *)(intptr_t)shield);
107 } else {
108 uint64_t where;
109 where = xc_unicast(0, interrupt_shield_xcall, ci,
110 (void *)(intptr_t)shield, ci);
111 xc_wait(where);
112 }
113
114 spc->spc_lastmod = time_second;
115 return 0;
116}
117
118/*
119 * Move all assigned interrupts from "cpu_idx" to the other cpu as possible.
120 * The destination cpu is the lowest cpuid of available cpus.
121 * If there are no available cpus, give up to move interrupts.
122 */
123static int
124interrupt_avert_intr(u_int cpu_idx)
125{
126 kcpuset_t *cpuset;
127 struct intrids_handler *ii_handler;
128 intrid_t *ids;
129 int error, i, nids;
130
131 kcpuset_create(&cpuset, true);
132 kcpuset_set(cpuset, cpu_idx);
133
134 ii_handler = interrupt_construct_intrids(cpuset);
135 if (ii_handler == NULL) {
136 error = ENOMEM;
137 goto out;
138 }
139 nids = ii_handler->iih_nids;
140 if (nids == 0) {
141 error = 0;
142 goto destruct_out;
143 }
144
145 interrupt_get_available(cpuset);
146 kcpuset_clear(cpuset, cpu_idx);
147 if (kcpuset_iszero(cpuset)) {
148 DPRINTF(("%s: no available cpu\n", __func__));
149 error = ENOENT;
150 goto destruct_out;
151 }
152
153 ids = ii_handler->iih_intrids;
154 for (i = 0; i < nids; i++) {
155 error = interrupt_distribute_handler(ids[i], cpuset, NULL);
156 if (error)
157 break;
158 }
159
160 destruct_out:
161 interrupt_destruct_intrids(ii_handler);
162 out:
163 kcpuset_destroy(cpuset);
164 return error;
165}
166
167/*
168 * Return actual intrio_list_line size.
169 * intrio_list_line size is variable by ncpu.
170 */
171static size_t
172interrupt_intrio_list_line_size(void)
173{
174
175 return sizeof(struct intrio_list_line) +
176 sizeof(struct intrio_list_line_cpu) * (ncpu - 1);
177}
178
179/*
180 * Return the size of interrupts list data on success.
181 * Reterun 0 on failed.
182 */
183static size_t
184interrupt_intrio_list_size(void)
185{
186 struct intrids_handler *ii_handler;
187 size_t ilsize;
188
189 ilsize = 0;
190
191 /* buffer header */
192 ilsize += sizeof(struct intrio_list);
193
194 /* il_line body */
195 ii_handler = interrupt_construct_intrids(kcpuset_running);
196 if (ii_handler == NULL)
197 return 0;
198 ilsize += interrupt_intrio_list_line_size() * (ii_handler->iih_nids);
199
200 interrupt_destruct_intrids(ii_handler);
201 return ilsize;
202}
203
204/*
205 * Set intrctl list data to "il", and return list structure bytes.
206 * If error occured, return <0.
207 * If "data" == NULL, simply return list structure bytes.
208 */
209static int
210interrupt_intrio_list(struct intrio_list *il, int length)
211{
212 struct intrio_list_line *illine;
213 kcpuset_t *assigned, *avail;
214 struct intrids_handler *ii_handler;
215 intrid_t *ids;
216 size_t ilsize;
217 u_int cpu_idx;
218 int nids, intr_idx, ret, line_size;
219
220 ilsize = interrupt_intrio_list_size();
221 if (ilsize == 0)
222 return -ENOMEM;
223
224 if (il == NULL)
225 return ilsize;
226
227 if (length < ilsize)
228 return -ENOMEM;
229
230 illine = (struct intrio_list_line *)
231 ((char *)il + sizeof(struct intrio_list));
232 il->il_lineoffset = (off_t)((uintptr_t)illine - (uintptr_t)il);
233
234 kcpuset_create(&avail, true);
235 interrupt_get_available(avail);
236 kcpuset_create(&assigned, true);
237
238 ii_handler = interrupt_construct_intrids(kcpuset_running);
239 if (ii_handler == NULL) {
240 DPRINTF(("%s: interrupt_construct_intrids() failed\n",
241 __func__));
242 ret = -ENOMEM;
243 goto out;
244 }
245
246 line_size = interrupt_intrio_list_line_size();
247 /* ensure interrupts are not added after interrupt_intrio_list_size(). */
248 nids = ii_handler->iih_nids;
249 ids = ii_handler->iih_intrids;
250 if (ilsize < sizeof(struct intrio_list) + line_size * nids) {
251 DPRINTF(("%s: interrupts are added during execution.\n",
252 __func__));
253 ret = -ENOMEM;
254 goto destruct_out;
255 }
256
257 for (intr_idx = 0; intr_idx < nids; intr_idx++) {
258 char devname[INTRDEVNAMEBUF];
259
260 strncpy(illine->ill_intrid, ids[intr_idx], INTRIDBUF);
261 interrupt_get_devname(ids[intr_idx], devname, sizeof(devname));
262 strncpy(illine->ill_xname, devname, INTRDEVNAMEBUF);
263
264 interrupt_get_assigned(ids[intr_idx], assigned);
265 for (cpu_idx = 0; cpu_idx < ncpu; cpu_idx++) {
266 struct intrio_list_line_cpu *illcpu =
267 &illine->ill_cpu[cpu_idx];
268
269 illcpu->illc_assigned =
270 kcpuset_isset(assigned, cpu_idx) ? true : false;
271 illcpu->illc_count =
272 interrupt_get_count(ids[intr_idx], cpu_idx);
273 }
274
275 illine = (struct intrio_list_line *)
276 ((char *)illine + line_size);
277 }
278
279 ret = ilsize;
280 il->il_version = INTRIO_LIST_VERSION;
281 il->il_ncpus = ncpu;
282 il->il_nintrs = nids;
283 il->il_linesize = line_size;
284 il->il_bufsize = ilsize;
285
286 destruct_out:
287 interrupt_destruct_intrids(ii_handler);
288 out:
289 kcpuset_destroy(assigned);
290 kcpuset_destroy(avail);
291
292 return ret;
293}
294
295/*
296 * "intrctl list" entry
297 */
298static int
299interrupt_intrio_list_sysctl(SYSCTLFN_ARGS)
300{
301 int ret, error;
302 void *buf;
303
304 if (oldlenp == NULL)
305 return EINVAL;
306
307 /*
308 * If oldp == NULL, the sysctl(8) caller process want to get the size of
309 * intrctl list data only.
310 */
311 if (oldp == NULL) {
312 ret = interrupt_intrio_list(NULL, 0);
313 if (ret < 0)
314 return -ret;
315
316 *oldlenp = ret;
317 return 0;
318 }
319
320 /*
321 * If oldp != NULL, the sysctl(8) caller process want to get both the size
322 * and the contents of intrctl list data.
323 */
324 if (*oldlenp == 0)
325 return ENOMEM;
326
327 buf = kmem_zalloc(*oldlenp, KM_SLEEP);
328 if (buf == NULL)
329 return ENOMEM;
330
331 ret = interrupt_intrio_list(buf, *oldlenp);
332 if (ret < 0) {
333 error = -ret;
334 goto out;
335 }
336 error = copyout(buf, oldp, *oldlenp);
337
338 out:
339 kmem_free(buf, *oldlenp);
340 return error;
341}
342
343/*
344 * "intrctl affinity" entry
345 */
346static int
347interrupt_set_affinity_sysctl(SYSCTLFN_ARGS)
348{
349 struct sysctlnode node;
350 struct intrio_set *iset;
351 cpuset_t *ucpuset;
352 kcpuset_t *kcpuset;
353 int error;
354
355 error = kauth_authorize_system(l->l_cred, KAUTH_SYSTEM_INTR,
356 KAUTH_REQ_SYSTEM_INTR_AFFINITY, NULL, NULL, NULL);
357 if (error)
358 return EPERM;
359
360 node = *rnode;
361 iset = (struct intrio_set *)node.sysctl_data;
362
363 error = sysctl_lookup(SYSCTLFN_CALL(&node));
364 if (error != 0 || newp == NULL)
365 return error;
366
367 ucpuset = iset->cpuset;
368 kcpuset_create(&kcpuset, true);
369 error = kcpuset_copyin(ucpuset, kcpuset, iset->cpuset_size);
370 if (error)
371 goto out;
372 if (kcpuset_iszero(kcpuset)) {
373 error = EINVAL;
374 goto out;
375 }
376
377 error = interrupt_distribute_handler(iset->intrid, kcpuset, NULL);
378
379 out:
380 kcpuset_destroy(kcpuset);
381 return error;
382}
383
384/*
385 * "intrctl intr" entry
386 */
387static int
388interrupt_intr_sysctl(SYSCTLFN_ARGS)
389{
390 struct sysctlnode node;
391 struct intrio_set *iset;
392 cpuset_t *ucpuset;
393 kcpuset_t *kcpuset;
394 int error;
395 u_int cpu_idx;
396
397 error = kauth_authorize_system(l->l_cred, KAUTH_SYSTEM_CPU,
398 KAUTH_REQ_SYSTEM_CPU_SETSTATE, NULL, NULL, NULL);
399 if (error)
400 return EPERM;
401
402 node = *rnode;
403 iset = (struct intrio_set *)node.sysctl_data;
404
405 error = sysctl_lookup(SYSCTLFN_CALL(&node));
406 if (error != 0 || newp == NULL)
407 return error;
408
409 ucpuset = iset->cpuset;
410 kcpuset_create(&kcpuset, true);
411 error = kcpuset_copyin(ucpuset, kcpuset, iset->cpuset_size);
412 if (error)
413 goto out;
414 if (kcpuset_iszero(kcpuset)) {
415 error = EINVAL;
416 goto out;
417 }
418
419 cpu_idx = kcpuset_ffs(kcpuset) - 1; /* support one CPU only */
420
421 mutex_enter(&cpu_lock);
422 error = interrupt_shield(cpu_idx, UNSET_NOINTR_SHIELD);
423 mutex_exit(&cpu_lock);
424
425 out:
426 kcpuset_destroy(kcpuset);
427 return error;
428}
429
430/*
431 * "intrctl nointr" entry
432 */
433static int
434interrupt_nointr_sysctl(SYSCTLFN_ARGS)
435{
436 struct sysctlnode node;
437 struct intrio_set *iset;
438 cpuset_t *ucpuset;
439 kcpuset_t *kcpuset;
440 int error;
441 u_int cpu_idx;
442
443 error = kauth_authorize_system(l->l_cred, KAUTH_SYSTEM_CPU,
444 KAUTH_REQ_SYSTEM_CPU_SETSTATE, NULL, NULL, NULL);
445 if (error)
446 return EPERM;
447
448 node = *rnode;
449 iset = (struct intrio_set *)node.sysctl_data;
450
451 error = sysctl_lookup(SYSCTLFN_CALL(&node));
452 if (error != 0 || newp == NULL)
453 return error;
454
455 ucpuset = iset->cpuset;
456 kcpuset_create(&kcpuset, true);
457 error = kcpuset_copyin(ucpuset, kcpuset, iset->cpuset_size);
458 if (error)
459 goto out;
460 if (kcpuset_iszero(kcpuset)) {
461 error = EINVAL;
462 goto out;
463 }
464
465 cpu_idx = kcpuset_ffs(kcpuset) - 1; /* support one CPU only */
466
467 mutex_enter(&cpu_lock);
468 error = interrupt_shield(cpu_idx, SET_NOINTR_SHIELD);
469 mutex_exit(&cpu_lock);
470 if (error)
471 goto out;
472
473 error = interrupt_avert_intr(cpu_idx);
474
475 out:
476 kcpuset_destroy(kcpuset);
477 return error;
478}
479
480SYSCTL_SETUP(sysctl_interrupt_setup, "sysctl interrupt setup")
481{
482 const struct sysctlnode *node = NULL;
483
484 sysctl_createv(clog, 0, NULL, &node,
485 CTLFLAG_PERMANENT, CTLTYPE_NODE,
486 "intr", SYSCTL_DESCR("Interrupt options"),
487 NULL, 0, NULL, 0,
488 CTL_KERN, CTL_CREATE, CTL_EOL);
489
490 sysctl_createv(clog, 0, &node, NULL,
491 CTLFLAG_PERMANENT, CTLTYPE_STRUCT,
492 "list", SYSCTL_DESCR("intrctl list"),
493 interrupt_intrio_list_sysctl, 0, NULL,
494 0, CTL_CREATE, CTL_EOL);
495
496 sysctl_createv(clog, 0, &node, NULL,
497 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_STRUCT,
498 "affinity", SYSCTL_DESCR("set affinity"),
499 interrupt_set_affinity_sysctl, 0, &kintrio_set,
500 sizeof(kintrio_set), CTL_CREATE, CTL_EOL);
501
502 sysctl_createv(clog, 0, &node, NULL,
503 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_STRUCT,
504 "intr", SYSCTL_DESCR("set intr"),
505 interrupt_intr_sysctl, 0, &kintrio_set,
506 sizeof(kintrio_set), CTL_CREATE, CTL_EOL);
507
508 sysctl_createv(clog, 0, &node, NULL,
509 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_STRUCT,
510 "nointr", SYSCTL_DESCR("set nointr"),
511 interrupt_nointr_sysctl, 0, &kintrio_set,
512 sizeof(kintrio_set), CTL_CREATE, CTL_EOL);
513}
514