1/* $NetBSD: kern_synch.c,v 1.311 2016/07/03 14:24:58 christos Exp $ */
2
3/*-
4 * Copyright (c) 1999, 2000, 2004, 2006, 2007, 2008, 2009
5 * The NetBSD Foundation, Inc.
6 * All rights reserved.
7 *
8 * This code is derived from software contributed to The NetBSD Foundation
9 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
10 * NASA Ames Research Center, by Charles M. Hannum, Andrew Doran and
11 * Daniel Sieger.
12 *
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
15 * are met:
16 * 1. Redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer.
18 * 2. Redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
23 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
24 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
25 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
26 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
27 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
28 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
29 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
30 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
31 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
32 * POSSIBILITY OF SUCH DAMAGE.
33 */
34
35/*-
36 * Copyright (c) 1982, 1986, 1990, 1991, 1993
37 * The Regents of the University of California. All rights reserved.
38 * (c) UNIX System Laboratories, Inc.
39 * All or some portions of this file are derived from material licensed
40 * to the University of California by American Telephone and Telegraph
41 * Co. or Unix System Laboratories, Inc. and are reproduced herein with
42 * the permission of UNIX System Laboratories, Inc.
43 *
44 * Redistribution and use in source and binary forms, with or without
45 * modification, are permitted provided that the following conditions
46 * are met:
47 * 1. Redistributions of source code must retain the above copyright
48 * notice, this list of conditions and the following disclaimer.
49 * 2. Redistributions in binary form must reproduce the above copyright
50 * notice, this list of conditions and the following disclaimer in the
51 * documentation and/or other materials provided with the distribution.
52 * 3. Neither the name of the University nor the names of its contributors
53 * may be used to endorse or promote products derived from this software
54 * without specific prior written permission.
55 *
56 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
57 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
58 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
59 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
60 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
61 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
62 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
63 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
64 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
65 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
66 * SUCH DAMAGE.
67 *
68 * @(#)kern_synch.c 8.9 (Berkeley) 5/19/95
69 */
70
71#include <sys/cdefs.h>
72__KERNEL_RCSID(0, "$NetBSD: kern_synch.c,v 1.311 2016/07/03 14:24:58 christos Exp $");
73
74#include "opt_kstack.h"
75#include "opt_perfctrs.h"
76#include "opt_dtrace.h"
77
78#define __MUTEX_PRIVATE
79
80#include <sys/param.h>
81#include <sys/systm.h>
82#include <sys/proc.h>
83#include <sys/kernel.h>
84#if defined(PERFCTRS)
85#include <sys/pmc.h>
86#endif
87#include <sys/cpu.h>
88#include <sys/pserialize.h>
89#include <sys/resourcevar.h>
90#include <sys/sched.h>
91#include <sys/syscall_stats.h>
92#include <sys/sleepq.h>
93#include <sys/lockdebug.h>
94#include <sys/evcnt.h>
95#include <sys/intr.h>
96#include <sys/lwpctl.h>
97#include <sys/atomic.h>
98#include <sys/syslog.h>
99
100#include <uvm/uvm_extern.h>
101
102#include <dev/lockstat.h>
103
104#include <sys/dtrace_bsd.h>
105int dtrace_vtime_active=0;
106dtrace_vtime_switch_func_t dtrace_vtime_switch_func;
107
108static void sched_unsleep(struct lwp *, bool);
109static void sched_changepri(struct lwp *, pri_t);
110static void sched_lendpri(struct lwp *, pri_t);
111static void resched_cpu(struct lwp *);
112
113syncobj_t sleep_syncobj = {
114 SOBJ_SLEEPQ_SORTED,
115 sleepq_unsleep,
116 sleepq_changepri,
117 sleepq_lendpri,
118 syncobj_noowner,
119};
120
121syncobj_t sched_syncobj = {
122 SOBJ_SLEEPQ_SORTED,
123 sched_unsleep,
124 sched_changepri,
125 sched_lendpri,
126 syncobj_noowner,
127};
128
129/* "Lightning bolt": once a second sleep address. */
130kcondvar_t lbolt __cacheline_aligned;
131
132u_int sched_pstats_ticks __cacheline_aligned;
133
134/* Preemption event counters. */
135static struct evcnt kpreempt_ev_crit __cacheline_aligned;
136static struct evcnt kpreempt_ev_klock __cacheline_aligned;
137static struct evcnt kpreempt_ev_immed __cacheline_aligned;
138
139void
140synch_init(void)
141{
142
143 cv_init(&lbolt, "lbolt");
144
145 evcnt_attach_dynamic(&kpreempt_ev_crit, EVCNT_TYPE_MISC, NULL,
146 "kpreempt", "defer: critical section");
147 evcnt_attach_dynamic(&kpreempt_ev_klock, EVCNT_TYPE_MISC, NULL,
148 "kpreempt", "defer: kernel_lock");
149 evcnt_attach_dynamic(&kpreempt_ev_immed, EVCNT_TYPE_MISC, NULL,
150 "kpreempt", "immediate");
151}
152
153/*
154 * OBSOLETE INTERFACE
155 *
156 * General sleep call. Suspends the current LWP until a wakeup is
157 * performed on the specified identifier. The LWP will then be made
158 * runnable with the specified priority. Sleeps at most timo/hz seconds (0
159 * means no timeout). If pri includes PCATCH flag, signals are checked
160 * before and after sleeping, else signals are not checked. Returns 0 if
161 * awakened, EWOULDBLOCK if the timeout expires. If PCATCH is set and a
162 * signal needs to be delivered, ERESTART is returned if the current system
163 * call should be restarted if possible, and EINTR is returned if the system
164 * call should be interrupted by the signal (return EINTR).
165 */
166int
167tsleep(wchan_t ident, pri_t priority, const char *wmesg, int timo)
168{
169 struct lwp *l = curlwp;
170 sleepq_t *sq;
171 kmutex_t *mp;
172
173 KASSERT((l->l_pflag & LP_INTR) == 0);
174 KASSERT(ident != &lbolt);
175
176 if (sleepq_dontsleep(l)) {
177 (void)sleepq_abort(NULL, 0);
178 return 0;
179 }
180
181 l->l_kpriority = true;
182 sq = sleeptab_lookup(&sleeptab, ident, &mp);
183 sleepq_enter(sq, l, mp);
184 sleepq_enqueue(sq, ident, wmesg, &sleep_syncobj);
185 return sleepq_block(timo, priority & PCATCH);
186}
187
188int
189mtsleep(wchan_t ident, pri_t priority, const char *wmesg, int timo,
190 kmutex_t *mtx)
191{
192 struct lwp *l = curlwp;
193 sleepq_t *sq;
194 kmutex_t *mp;
195 int error;
196
197 KASSERT((l->l_pflag & LP_INTR) == 0);
198 KASSERT(ident != &lbolt);
199
200 if (sleepq_dontsleep(l)) {
201 (void)sleepq_abort(mtx, (priority & PNORELOCK) != 0);
202 return 0;
203 }
204
205 l->l_kpriority = true;
206 sq = sleeptab_lookup(&sleeptab, ident, &mp);
207 sleepq_enter(sq, l, mp);
208 sleepq_enqueue(sq, ident, wmesg, &sleep_syncobj);
209 mutex_exit(mtx);
210 error = sleepq_block(timo, priority & PCATCH);
211
212 if ((priority & PNORELOCK) == 0)
213 mutex_enter(mtx);
214
215 return error;
216}
217
218/*
219 * General sleep call for situations where a wake-up is not expected.
220 */
221int
222kpause(const char *wmesg, bool intr, int timo, kmutex_t *mtx)
223{
224 struct lwp *l = curlwp;
225 kmutex_t *mp;
226 sleepq_t *sq;
227 int error;
228
229 KASSERT(!(timo == 0 && intr == false));
230
231 if (sleepq_dontsleep(l))
232 return sleepq_abort(NULL, 0);
233
234 if (mtx != NULL)
235 mutex_exit(mtx);
236 l->l_kpriority = true;
237 sq = sleeptab_lookup(&sleeptab, l, &mp);
238 sleepq_enter(sq, l, mp);
239 sleepq_enqueue(sq, l, wmesg, &sleep_syncobj);
240 error = sleepq_block(timo, intr);
241 if (mtx != NULL)
242 mutex_enter(mtx);
243
244 return error;
245}
246
247/*
248 * OBSOLETE INTERFACE
249 *
250 * Make all LWPs sleeping on the specified identifier runnable.
251 */
252void
253wakeup(wchan_t ident)
254{
255 sleepq_t *sq;
256 kmutex_t *mp;
257
258 if (__predict_false(cold))
259 return;
260
261 sq = sleeptab_lookup(&sleeptab, ident, &mp);
262 sleepq_wake(sq, ident, (u_int)-1, mp);
263}
264
265/*
266 * General yield call. Puts the current LWP back on its run queue and
267 * performs a voluntary context switch. Should only be called when the
268 * current LWP explicitly requests it (eg sched_yield(2)).
269 */
270void
271yield(void)
272{
273 struct lwp *l = curlwp;
274
275 KERNEL_UNLOCK_ALL(l, &l->l_biglocks);
276 lwp_lock(l);
277 KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_lwplock));
278 KASSERT(l->l_stat == LSONPROC);
279 l->l_kpriority = false;
280 (void)mi_switch(l);
281 KERNEL_LOCK(l->l_biglocks, l);
282}
283
284/*
285 * General preemption call. Puts the current LWP back on its run queue
286 * and performs an involuntary context switch.
287 */
288void
289preempt(void)
290{
291 struct lwp *l = curlwp;
292
293 KERNEL_UNLOCK_ALL(l, &l->l_biglocks);
294 lwp_lock(l);
295 KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_lwplock));
296 KASSERT(l->l_stat == LSONPROC);
297 l->l_kpriority = false;
298 l->l_nivcsw++;
299 (void)mi_switch(l);
300 KERNEL_LOCK(l->l_biglocks, l);
301}
302
303/*
304 * Handle a request made by another agent to preempt the current LWP
305 * in-kernel. Usually called when l_dopreempt may be non-zero.
306 *
307 * Character addresses for lockstat only.
308 */
309static char in_critical_section;
310static char kernel_lock_held;
311static char is_softint;
312static char cpu_kpreempt_enter_fail;
313
314bool
315kpreempt(uintptr_t where)
316{
317 uintptr_t failed;
318 lwp_t *l;
319 int s, dop, lsflag;
320
321 l = curlwp;
322 failed = 0;
323 while ((dop = l->l_dopreempt) != 0) {
324 if (l->l_stat != LSONPROC) {
325 /*
326 * About to block (or die), let it happen.
327 * Doesn't really count as "preemption has
328 * been blocked", since we're going to
329 * context switch.
330 */
331 l->l_dopreempt = 0;
332 return true;
333 }
334 if (__predict_false((l->l_flag & LW_IDLE) != 0)) {
335 /* Can't preempt idle loop, don't count as failure. */
336 l->l_dopreempt = 0;
337 return true;
338 }
339 if (__predict_false(l->l_nopreempt != 0)) {
340 /* LWP holds preemption disabled, explicitly. */
341 if ((dop & DOPREEMPT_COUNTED) == 0) {
342 kpreempt_ev_crit.ev_count++;
343 }
344 failed = (uintptr_t)&in_critical_section;
345 break;
346 }
347 if (__predict_false((l->l_pflag & LP_INTR) != 0)) {
348 /* Can't preempt soft interrupts yet. */
349 l->l_dopreempt = 0;
350 failed = (uintptr_t)&is_softint;
351 break;
352 }
353 s = splsched();
354 if (__predict_false(l->l_blcnt != 0 ||
355 curcpu()->ci_biglock_wanted != NULL)) {
356 /* Hold or want kernel_lock, code is not MT safe. */
357 splx(s);
358 if ((dop & DOPREEMPT_COUNTED) == 0) {
359 kpreempt_ev_klock.ev_count++;
360 }
361 failed = (uintptr_t)&kernel_lock_held;
362 break;
363 }
364 if (__predict_false(!cpu_kpreempt_enter(where, s))) {
365 /*
366 * It may be that the IPL is too high.
367 * kpreempt_enter() can schedule an
368 * interrupt to retry later.
369 */
370 splx(s);
371 failed = (uintptr_t)&cpu_kpreempt_enter_fail;
372 break;
373 }
374 /* Do it! */
375 if (__predict_true((dop & DOPREEMPT_COUNTED) == 0)) {
376 kpreempt_ev_immed.ev_count++;
377 }
378 lwp_lock(l);
379 mi_switch(l);
380 l->l_nopreempt++;
381 splx(s);
382
383 /* Take care of any MD cleanup. */
384 cpu_kpreempt_exit(where);
385 l->l_nopreempt--;
386 }
387
388 if (__predict_true(!failed)) {
389 return false;
390 }
391
392 /* Record preemption failure for reporting via lockstat. */
393 atomic_or_uint(&l->l_dopreempt, DOPREEMPT_COUNTED);
394 lsflag = 0;
395 LOCKSTAT_ENTER(lsflag);
396 if (__predict_false(lsflag)) {
397 if (where == 0) {
398 where = (uintptr_t)__builtin_return_address(0);
399 }
400 /* Preemption is on, might recurse, so make it atomic. */
401 if (atomic_cas_ptr_ni((void *)&l->l_pfailaddr, NULL,
402 (void *)where) == NULL) {
403 LOCKSTAT_START_TIMER(lsflag, l->l_pfailtime);
404 l->l_pfaillock = failed;
405 }
406 }
407 LOCKSTAT_EXIT(lsflag);
408 return true;
409}
410
411/*
412 * Return true if preemption is explicitly disabled.
413 */
414bool
415kpreempt_disabled(void)
416{
417 const lwp_t *l = curlwp;
418
419 return l->l_nopreempt != 0 || l->l_stat == LSZOMB ||
420 (l->l_flag & LW_IDLE) != 0 || cpu_kpreempt_disabled();
421}
422
423/*
424 * Disable kernel preemption.
425 */
426void
427kpreempt_disable(void)
428{
429
430 KPREEMPT_DISABLE(curlwp);
431}
432
433/*
434 * Reenable kernel preemption.
435 */
436void
437kpreempt_enable(void)
438{
439
440 KPREEMPT_ENABLE(curlwp);
441}
442
443/*
444 * Compute the amount of time during which the current lwp was running.
445 *
446 * - update l_rtime unless it's an idle lwp.
447 */
448
449void
450updatertime(lwp_t *l, const struct bintime *now)
451{
452
453 if (__predict_false(l->l_flag & LW_IDLE))
454 return;
455
456 /* rtime += now - stime */
457 bintime_add(&l->l_rtime, now);
458 bintime_sub(&l->l_rtime, &l->l_stime);
459}
460
461/*
462 * Select next LWP from the current CPU to run..
463 */
464static inline lwp_t *
465nextlwp(struct cpu_info *ci, struct schedstate_percpu *spc)
466{
467 lwp_t *newl;
468
469 /*
470 * Let sched_nextlwp() select the LWP to run the CPU next.
471 * If no LWP is runnable, select the idle LWP.
472 *
473 * Note that spc_lwplock might not necessary be held, and
474 * new thread would be unlocked after setting the LWP-lock.
475 */
476 newl = sched_nextlwp();
477 if (newl != NULL) {
478 sched_dequeue(newl);
479 KASSERT(lwp_locked(newl, spc->spc_mutex));
480 KASSERT(newl->l_cpu == ci);
481 newl->l_stat = LSONPROC;
482 newl->l_pflag |= LP_RUNNING;
483 lwp_setlock(newl, spc->spc_lwplock);
484 } else {
485 newl = ci->ci_data.cpu_idlelwp;
486 newl->l_stat = LSONPROC;
487 newl->l_pflag |= LP_RUNNING;
488 }
489
490 /*
491 * Only clear want_resched if there are no pending (slow)
492 * software interrupts.
493 */
494 ci->ci_want_resched = ci->ci_data.cpu_softints;
495 spc->spc_flags &= ~SPCF_SWITCHCLEAR;
496 spc->spc_curpriority = lwp_eprio(newl);
497
498 return newl;
499}
500
501/*
502 * The machine independent parts of context switch.
503 *
504 * Returns 1 if another LWP was actually run.
505 */
506int
507mi_switch(lwp_t *l)
508{
509 struct cpu_info *ci;
510 struct schedstate_percpu *spc;
511 struct lwp *newl;
512 int retval, oldspl;
513 struct bintime bt;
514 bool returning;
515
516 KASSERT(lwp_locked(l, NULL));
517 KASSERT(kpreempt_disabled());
518 LOCKDEBUG_BARRIER(l->l_mutex, 1);
519
520 kstack_check_magic(l);
521
522 binuptime(&bt);
523
524 KASSERTMSG(l == curlwp, "l %p curlwp %p", l, curlwp);
525 KASSERT((l->l_pflag & LP_RUNNING) != 0);
526 KASSERT(l->l_cpu == curcpu());
527 ci = l->l_cpu;
528 spc = &ci->ci_schedstate;
529 returning = false;
530 newl = NULL;
531
532 /*
533 * If we have been asked to switch to a specific LWP, then there
534 * is no need to inspect the run queues. If a soft interrupt is
535 * blocking, then return to the interrupted thread without adjusting
536 * VM context or its start time: neither have been changed in order
537 * to take the interrupt.
538 */
539 if (l->l_switchto != NULL) {
540 if ((l->l_pflag & LP_INTR) != 0) {
541 returning = true;
542 softint_block(l);
543 if ((l->l_pflag & LP_TIMEINTR) != 0)
544 updatertime(l, &bt);
545 }
546 newl = l->l_switchto;
547 l->l_switchto = NULL;
548 }
549#ifndef __HAVE_FAST_SOFTINTS
550 else if (ci->ci_data.cpu_softints != 0) {
551 /* There are pending soft interrupts, so pick one. */
552 newl = softint_picklwp();
553 newl->l_stat = LSONPROC;
554 newl->l_pflag |= LP_RUNNING;
555 }
556#endif /* !__HAVE_FAST_SOFTINTS */
557
558 /* Count time spent in current system call */
559 if (!returning) {
560 SYSCALL_TIME_SLEEP(l);
561
562 /*
563 * XXXSMP If we are using h/w performance counters,
564 * save context.
565 */
566#if PERFCTRS
567 if (PMC_ENABLED(l->l_proc)) {
568 pmc_save_context(l->l_proc);
569 }
570#endif
571 updatertime(l, &bt);
572 }
573
574 /* Lock the runqueue */
575 KASSERT(l->l_stat != LSRUN);
576 mutex_spin_enter(spc->spc_mutex);
577
578 /*
579 * If on the CPU and we have gotten this far, then we must yield.
580 */
581 if (l->l_stat == LSONPROC && l != newl) {
582 KASSERT(lwp_locked(l, spc->spc_lwplock));
583 if ((l->l_flag & LW_IDLE) == 0) {
584 l->l_stat = LSRUN;
585 lwp_setlock(l, spc->spc_mutex);
586 sched_enqueue(l, true);
587 /*
588 * Handle migration. Note that "migrating LWP" may
589 * be reset here, if interrupt/preemption happens
590 * early in idle LWP.
591 */
592 if (l->l_target_cpu != NULL) {
593 KASSERT((l->l_pflag & LP_INTR) == 0);
594 spc->spc_migrating = l;
595 }
596 } else
597 l->l_stat = LSIDL;
598 }
599
600 /* Pick new LWP to run. */
601 if (newl == NULL) {
602 newl = nextlwp(ci, spc);
603 }
604
605 /* Items that must be updated with the CPU locked. */
606 if (!returning) {
607 /* Update the new LWP's start time. */
608 newl->l_stime = bt;
609
610 /*
611 * ci_curlwp changes when a fast soft interrupt occurs.
612 * We use cpu_onproc to keep track of which kernel or
613 * user thread is running 'underneath' the software
614 * interrupt. This is important for time accounting,
615 * itimers and forcing user threads to preempt (aston).
616 */
617 ci->ci_data.cpu_onproc = newl;
618 }
619
620 /*
621 * Preemption related tasks. Must be done with the current
622 * CPU locked.
623 */
624 cpu_did_resched(l);
625 l->l_dopreempt = 0;
626 if (__predict_false(l->l_pfailaddr != 0)) {
627 LOCKSTAT_FLAG(lsflag);
628 LOCKSTAT_ENTER(lsflag);
629 LOCKSTAT_STOP_TIMER(lsflag, l->l_pfailtime);
630 LOCKSTAT_EVENT_RA(lsflag, l->l_pfaillock, LB_NOPREEMPT|LB_SPIN,
631 1, l->l_pfailtime, l->l_pfailaddr);
632 LOCKSTAT_EXIT(lsflag);
633 l->l_pfailtime = 0;
634 l->l_pfaillock = 0;
635 l->l_pfailaddr = 0;
636 }
637
638 if (l != newl) {
639 struct lwp *prevlwp;
640
641 /* Release all locks, but leave the current LWP locked */
642 if (l->l_mutex == spc->spc_mutex) {
643 /*
644 * Drop spc_lwplock, if the current LWP has been moved
645 * to the run queue (it is now locked by spc_mutex).
646 */
647 mutex_spin_exit(spc->spc_lwplock);
648 } else {
649 /*
650 * Otherwise, drop the spc_mutex, we are done with the
651 * run queues.
652 */
653 mutex_spin_exit(spc->spc_mutex);
654 }
655
656 /*
657 * Mark that context switch is going to be performed
658 * for this LWP, to protect it from being switched
659 * to on another CPU.
660 */
661 KASSERT(l->l_ctxswtch == 0);
662 l->l_ctxswtch = 1;
663 l->l_ncsw++;
664 KASSERT((l->l_pflag & LP_RUNNING) != 0);
665 l->l_pflag &= ~LP_RUNNING;
666
667 /*
668 * Increase the count of spin-mutexes before the release
669 * of the last lock - we must remain at IPL_SCHED during
670 * the context switch.
671 */
672 KASSERTMSG(ci->ci_mtx_count == -1,
673 "%s: cpu%u: ci_mtx_count (%d) != -1 "
674 "(block with spin-mutex held)",
675 __func__, cpu_index(ci), ci->ci_mtx_count);
676 oldspl = MUTEX_SPIN_OLDSPL(ci);
677 ci->ci_mtx_count--;
678 lwp_unlock(l);
679
680 /* Count the context switch on this CPU. */
681 ci->ci_data.cpu_nswtch++;
682
683 /* Update status for lwpctl, if present. */
684 if (l->l_lwpctl != NULL)
685 l->l_lwpctl->lc_curcpu = LWPCTL_CPU_NONE;
686
687 /*
688 * Save old VM context, unless a soft interrupt
689 * handler is blocking.
690 */
691 if (!returning)
692 pmap_deactivate(l);
693
694 /*
695 * We may need to spin-wait if 'newl' is still
696 * context switching on another CPU.
697 */
698 if (__predict_false(newl->l_ctxswtch != 0)) {
699 u_int count;
700 count = SPINLOCK_BACKOFF_MIN;
701 while (newl->l_ctxswtch)
702 SPINLOCK_BACKOFF(count);
703 }
704
705 /*
706 * If DTrace has set the active vtime enum to anything
707 * other than INACTIVE (0), then it should have set the
708 * function to call.
709 */
710 if (__predict_false(dtrace_vtime_active)) {
711 (*dtrace_vtime_switch_func)(newl);
712 }
713
714 /* Switch to the new LWP.. */
715#ifdef MULTIPROCESSOR
716 KASSERT(curlwp == ci->ci_curlwp);
717#endif
718 KASSERTMSG(l == curlwp, "l %p curlwp %p", l, curlwp);
719 prevlwp = cpu_switchto(l, newl, returning);
720 ci = curcpu();
721#ifdef MULTIPROCESSOR
722 KASSERT(curlwp == ci->ci_curlwp);
723#endif
724 KASSERTMSG(l == curlwp, "l %p curlwp %p prevlwp %p",
725 l, curlwp, prevlwp);
726
727 /*
728 * Switched away - we have new curlwp.
729 * Restore VM context and IPL.
730 */
731 pmap_activate(l);
732 uvm_emap_switch(l);
733 pcu_switchpoint(l);
734
735 if (prevlwp != NULL) {
736 /* Normalize the count of the spin-mutexes */
737 ci->ci_mtx_count++;
738 /* Unmark the state of context switch */
739 membar_exit();
740 prevlwp->l_ctxswtch = 0;
741 }
742
743 /* Update status for lwpctl, if present. */
744 if (l->l_lwpctl != NULL) {
745 l->l_lwpctl->lc_curcpu = (int)cpu_index(ci);
746 l->l_lwpctl->lc_pctr++;
747 }
748
749 /* Note trip through cpu_switchto(). */
750 pserialize_switchpoint();
751
752 KASSERT(l->l_cpu == ci);
753 splx(oldspl);
754 /*
755 * note that, unless the caller disabled preemption,
756 * we can be preempted at any time after the above splx() call.
757 */
758 retval = 1;
759 } else {
760 /* Nothing to do - just unlock and return. */
761 mutex_spin_exit(spc->spc_mutex);
762 lwp_unlock(l);
763 retval = 0;
764 }
765
766 KASSERT(l == curlwp);
767 KASSERT(l->l_stat == LSONPROC);
768
769 /*
770 * XXXSMP If we are using h/w performance counters, restore context.
771 * XXXSMP preemption problem.
772 */
773#if PERFCTRS
774 if (PMC_ENABLED(l->l_proc)) {
775 pmc_restore_context(l->l_proc);
776 }
777#endif
778 SYSCALL_TIME_WAKEUP(l);
779 LOCKDEBUG_BARRIER(NULL, 1);
780
781 return retval;
782}
783
784/*
785 * The machine independent parts of context switch to oblivion.
786 * Does not return. Call with the LWP unlocked.
787 */
788void
789lwp_exit_switchaway(lwp_t *l)
790{
791 struct cpu_info *ci;
792 struct lwp *newl;
793 struct bintime bt;
794
795 ci = l->l_cpu;
796
797 KASSERT(kpreempt_disabled());
798 KASSERT(l->l_stat == LSZOMB || l->l_stat == LSIDL);
799 KASSERT(ci == curcpu());
800 LOCKDEBUG_BARRIER(NULL, 0);
801
802 kstack_check_magic(l);
803
804 /* Count time spent in current system call */
805 SYSCALL_TIME_SLEEP(l);
806 binuptime(&bt);
807 updatertime(l, &bt);
808
809 /* Must stay at IPL_SCHED even after releasing run queue lock. */
810 (void)splsched();
811
812 /*
813 * Let sched_nextlwp() select the LWP to run the CPU next.
814 * If no LWP is runnable, select the idle LWP.
815 *
816 * Note that spc_lwplock might not necessary be held, and
817 * new thread would be unlocked after setting the LWP-lock.
818 */
819 spc_lock(ci);
820#ifndef __HAVE_FAST_SOFTINTS
821 if (ci->ci_data.cpu_softints != 0) {
822 /* There are pending soft interrupts, so pick one. */
823 newl = softint_picklwp();
824 newl->l_stat = LSONPROC;
825 newl->l_pflag |= LP_RUNNING;
826 } else
827#endif /* !__HAVE_FAST_SOFTINTS */
828 {
829 newl = nextlwp(ci, &ci->ci_schedstate);
830 }
831
832 /* Update the new LWP's start time. */
833 newl->l_stime = bt;
834 l->l_pflag &= ~LP_RUNNING;
835
836 /*
837 * ci_curlwp changes when a fast soft interrupt occurs.
838 * We use cpu_onproc to keep track of which kernel or
839 * user thread is running 'underneath' the software
840 * interrupt. This is important for time accounting,
841 * itimers and forcing user threads to preempt (aston).
842 */
843 ci->ci_data.cpu_onproc = newl;
844
845 /*
846 * Preemption related tasks. Must be done with the current
847 * CPU locked.
848 */
849 cpu_did_resched(l);
850
851 /* Unlock the run queue. */
852 spc_unlock(ci);
853
854 /* Count the context switch on this CPU. */
855 ci->ci_data.cpu_nswtch++;
856
857 /* Update status for lwpctl, if present. */
858 if (l->l_lwpctl != NULL)
859 l->l_lwpctl->lc_curcpu = LWPCTL_CPU_EXITED;
860
861 /*
862 * We may need to spin-wait if 'newl' is still
863 * context switching on another CPU.
864 */
865 if (__predict_false(newl->l_ctxswtch != 0)) {
866 u_int count;
867 count = SPINLOCK_BACKOFF_MIN;
868 while (newl->l_ctxswtch)
869 SPINLOCK_BACKOFF(count);
870 }
871
872 /*
873 * If DTrace has set the active vtime enum to anything
874 * other than INACTIVE (0), then it should have set the
875 * function to call.
876 */
877 if (__predict_false(dtrace_vtime_active)) {
878 (*dtrace_vtime_switch_func)(newl);
879 }
880
881 /* Switch to the new LWP.. */
882 (void)cpu_switchto(NULL, newl, false);
883
884 for (;;) continue; /* XXX: convince gcc about "noreturn" */
885 /* NOTREACHED */
886}
887
888/*
889 * setrunnable: change LWP state to be runnable, placing it on the run queue.
890 *
891 * Call with the process and LWP locked. Will return with the LWP unlocked.
892 */
893void
894setrunnable(struct lwp *l)
895{
896 struct proc *p = l->l_proc;
897 struct cpu_info *ci;
898
899 KASSERT((l->l_flag & LW_IDLE) == 0);
900 KASSERT(mutex_owned(p->p_lock));
901 KASSERT(lwp_locked(l, NULL));
902 KASSERT(l->l_mutex != l->l_cpu->ci_schedstate.spc_mutex);
903
904 switch (l->l_stat) {
905 case LSSTOP:
906 /*
907 * If we're being traced (possibly because someone attached us
908 * while we were stopped), check for a signal from the debugger.
909 */
910 if ((p->p_slflag & PSL_TRACED) != 0 && p->p_xsig != 0)
911 signotify(l);
912 p->p_nrlwps++;
913 break;
914 case LSSUSPENDED:
915 l->l_flag &= ~LW_WSUSPEND;
916 p->p_nrlwps++;
917 cv_broadcast(&p->p_lwpcv);
918 break;
919 case LSSLEEP:
920 KASSERT(l->l_wchan != NULL);
921 break;
922 default:
923 panic("setrunnable: lwp %p state was %d", l, l->l_stat);
924 }
925
926 /*
927 * If the LWP was sleeping, start it again.
928 */
929 if (l->l_wchan != NULL) {
930 l->l_stat = LSSLEEP;
931 /* lwp_unsleep() will release the lock. */
932 lwp_unsleep(l, true);
933 return;
934 }
935
936 /*
937 * If the LWP is still on the CPU, mark it as LSONPROC. It may be
938 * about to call mi_switch(), in which case it will yield.
939 */
940 if ((l->l_pflag & LP_RUNNING) != 0) {
941 l->l_stat = LSONPROC;
942 l->l_slptime = 0;
943 lwp_unlock(l);
944 return;
945 }
946
947 /*
948 * Look for a CPU to run.
949 * Set the LWP runnable.
950 */
951 ci = sched_takecpu(l);
952 l->l_cpu = ci;
953 spc_lock(ci);
954 lwp_unlock_to(l, ci->ci_schedstate.spc_mutex);
955 sched_setrunnable(l);
956 l->l_stat = LSRUN;
957 l->l_slptime = 0;
958
959 sched_enqueue(l, false);
960 resched_cpu(l);
961 lwp_unlock(l);
962}
963
964/*
965 * suspendsched:
966 *
967 * Convert all non-LW_SYSTEM LSSLEEP or LSRUN LWPs to LSSUSPENDED.
968 */
969void
970suspendsched(void)
971{
972 CPU_INFO_ITERATOR cii;
973 struct cpu_info *ci;
974 struct lwp *l;
975 struct proc *p;
976
977 /*
978 * We do this by process in order not to violate the locking rules.
979 */
980 mutex_enter(proc_lock);
981 PROCLIST_FOREACH(p, &allproc) {
982 mutex_enter(p->p_lock);
983 if ((p->p_flag & PK_SYSTEM) != 0) {
984 mutex_exit(p->p_lock);
985 continue;
986 }
987
988 if (p->p_stat != SSTOP) {
989 if (p->p_stat != SZOMB && p->p_stat != SDEAD) {
990 p->p_pptr->p_nstopchild++;
991 p->p_waited = 0;
992 }
993 p->p_stat = SSTOP;
994 }
995
996 LIST_FOREACH(l, &p->p_lwps, l_sibling) {
997 if (l == curlwp)
998 continue;
999
1000 lwp_lock(l);
1001
1002 /*
1003 * Set L_WREBOOT so that the LWP will suspend itself
1004 * when it tries to return to user mode. We want to
1005 * try and get to get as many LWPs as possible to
1006 * the user / kernel boundary, so that they will
1007 * release any locks that they hold.
1008 */
1009 l->l_flag |= (LW_WREBOOT | LW_WSUSPEND);
1010
1011 if (l->l_stat == LSSLEEP &&
1012 (l->l_flag & LW_SINTR) != 0) {
1013 /* setrunnable() will release the lock. */
1014 setrunnable(l);
1015 continue;
1016 }
1017
1018 lwp_unlock(l);
1019 }
1020
1021 mutex_exit(p->p_lock);
1022 }
1023 mutex_exit(proc_lock);
1024
1025 /*
1026 * Kick all CPUs to make them preempt any LWPs running in user mode.
1027 * They'll trap into the kernel and suspend themselves in userret().
1028 */
1029 for (CPU_INFO_FOREACH(cii, ci)) {
1030 spc_lock(ci);
1031 cpu_need_resched(ci, RESCHED_IMMED);
1032 spc_unlock(ci);
1033 }
1034}
1035
1036/*
1037 * sched_unsleep:
1038 *
1039 * The is called when the LWP has not been awoken normally but instead
1040 * interrupted: for example, if the sleep timed out. Because of this,
1041 * it's not a valid action for running or idle LWPs.
1042 */
1043static void
1044sched_unsleep(struct lwp *l, bool cleanup)
1045{
1046
1047 lwp_unlock(l);
1048 panic("sched_unsleep");
1049}
1050
1051static void
1052resched_cpu(struct lwp *l)
1053{
1054 struct cpu_info *ci = l->l_cpu;
1055
1056 KASSERT(lwp_locked(l, NULL));
1057 if (lwp_eprio(l) > ci->ci_schedstate.spc_curpriority)
1058 cpu_need_resched(ci, 0);
1059}
1060
1061static void
1062sched_changepri(struct lwp *l, pri_t pri)
1063{
1064
1065 KASSERT(lwp_locked(l, NULL));
1066
1067 if (l->l_stat == LSRUN) {
1068 KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_mutex));
1069 sched_dequeue(l);
1070 l->l_priority = pri;
1071 sched_enqueue(l, false);
1072 } else {
1073 l->l_priority = pri;
1074 }
1075 resched_cpu(l);
1076}
1077
1078static void
1079sched_lendpri(struct lwp *l, pri_t pri)
1080{
1081
1082 KASSERT(lwp_locked(l, NULL));
1083
1084 if (l->l_stat == LSRUN) {
1085 KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_mutex));
1086 sched_dequeue(l);
1087 l->l_inheritedprio = pri;
1088 l->l_auxprio = MAX(l->l_inheritedprio, l->l_protectprio);
1089 sched_enqueue(l, false);
1090 } else {
1091 l->l_inheritedprio = pri;
1092 l->l_auxprio = MAX(l->l_inheritedprio, l->l_protectprio);
1093 }
1094 resched_cpu(l);
1095}
1096
1097struct lwp *
1098syncobj_noowner(wchan_t wchan)
1099{
1100
1101 return NULL;
1102}
1103
1104/* Decay 95% of proc::p_pctcpu in 60 seconds, ccpu = exp(-1/20) */
1105const fixpt_t ccpu = 0.95122942450071400909 * FSCALE;
1106
1107/*
1108 * Constants for averages over 1, 5 and 15 minutes when sampling at
1109 * 5 second intervals.
1110 */
1111static const fixpt_t cexp[ ] = {
1112 0.9200444146293232 * FSCALE, /* exp(-1/12) */
1113 0.9834714538216174 * FSCALE, /* exp(-1/60) */
1114 0.9944598480048967 * FSCALE, /* exp(-1/180) */
1115};
1116
1117/*
1118 * sched_pstats:
1119 *
1120 * => Update process statistics and check CPU resource allocation.
1121 * => Call scheduler-specific hook to eventually adjust LWP priorities.
1122 * => Compute load average of a quantity on 1, 5 and 15 minute intervals.
1123 */
1124void
1125sched_pstats(void)
1126{
1127 extern struct loadavg averunnable;
1128 struct loadavg *avg = &averunnable;
1129 const int clkhz = (stathz != 0 ? stathz : hz);
1130 static bool backwards = false;
1131 static u_int lavg_count = 0;
1132 struct proc *p;
1133 int nrun;
1134
1135 sched_pstats_ticks++;
1136 if (++lavg_count >= 5) {
1137 lavg_count = 0;
1138 nrun = 0;
1139 }
1140 mutex_enter(proc_lock);
1141 PROCLIST_FOREACH(p, &allproc) {
1142 struct lwp *l;
1143 struct rlimit *rlim;
1144 time_t runtm;
1145 int sig;
1146
1147 /* Increment sleep time (if sleeping), ignore overflow. */
1148 mutex_enter(p->p_lock);
1149 runtm = p->p_rtime.sec;
1150 LIST_FOREACH(l, &p->p_lwps, l_sibling) {
1151 fixpt_t lpctcpu;
1152 u_int lcpticks;
1153
1154 if (__predict_false((l->l_flag & LW_IDLE) != 0))
1155 continue;
1156 lwp_lock(l);
1157 runtm += l->l_rtime.sec;
1158 l->l_swtime++;
1159 sched_lwp_stats(l);
1160
1161 /* For load average calculation. */
1162 if (__predict_false(lavg_count == 0) &&
1163 (l->l_flag & (LW_SINTR | LW_SYSTEM)) == 0) {
1164 switch (l->l_stat) {
1165 case LSSLEEP:
1166 if (l->l_slptime > 1) {
1167 break;
1168 }
1169 case LSRUN:
1170 case LSONPROC:
1171 case LSIDL:
1172 nrun++;
1173 }
1174 }
1175 lwp_unlock(l);
1176
1177 l->l_pctcpu = (l->l_pctcpu * ccpu) >> FSHIFT;
1178 if (l->l_slptime != 0)
1179 continue;
1180
1181 lpctcpu = l->l_pctcpu;
1182 lcpticks = atomic_swap_uint(&l->l_cpticks, 0);
1183 lpctcpu += ((FSCALE - ccpu) *
1184 (lcpticks * FSCALE / clkhz)) >> FSHIFT;
1185 l->l_pctcpu = lpctcpu;
1186 }
1187 /* Calculating p_pctcpu only for ps(1) */
1188 p->p_pctcpu = (p->p_pctcpu * ccpu) >> FSHIFT;
1189
1190 if (__predict_false(runtm < 0)) {
1191 if (!backwards) {
1192 backwards = true;
1193 printf("WARNING: negative runtime; "
1194 "monotonic clock has gone backwards\n");
1195 }
1196 mutex_exit(p->p_lock);
1197 continue;
1198 }
1199
1200 /*
1201 * Check if the process exceeds its CPU resource allocation.
1202 * If over the hard limit, kill it with SIGKILL.
1203 * If over the soft limit, send SIGXCPU and raise
1204 * the soft limit a little.
1205 */
1206 rlim = &p->p_rlimit[RLIMIT_CPU];
1207 sig = 0;
1208 if (__predict_false(runtm >= rlim->rlim_cur)) {
1209 if (runtm >= rlim->rlim_max) {
1210 sig = SIGKILL;
1211 log(LOG_NOTICE, "pid %d is killed: %s\n",
1212 p->p_pid, "exceeded RLIMIT_CPU");
1213 uprintf("pid %d, command %s, is killed: %s\n",
1214 p->p_pid, p->p_comm,
1215 "exceeded RLIMIT_CPU");
1216 } else {
1217 sig = SIGXCPU;
1218 if (rlim->rlim_cur < rlim->rlim_max)
1219 rlim->rlim_cur += 5;
1220 }
1221 }
1222 mutex_exit(p->p_lock);
1223 if (__predict_false(sig)) {
1224 KASSERT((p->p_flag & PK_SYSTEM) == 0);
1225 psignal(p, sig);
1226 }
1227 }
1228 mutex_exit(proc_lock);
1229
1230 /* Load average calculation. */
1231 if (__predict_false(lavg_count == 0)) {
1232 int i;
1233 CTASSERT(__arraycount(cexp) == __arraycount(avg->ldavg));
1234 for (i = 0; i < __arraycount(cexp); i++) {
1235 avg->ldavg[i] = (cexp[i] * avg->ldavg[i] +
1236 nrun * FSCALE * (FSCALE - cexp[i])) >> FSHIFT;
1237 }
1238 }
1239
1240 /* Lightning bolt. */
1241 cv_broadcast(&lbolt);
1242}
1243