1/* $NetBSD: lapic.c,v 1.53 2016/10/15 09:50:27 maxv Exp $ */
2
3/*-
4 * Copyright (c) 2000, 2008 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by RedBack Networks Inc.
9 *
10 * Author: Bill Sommerfeld
11 *
12 * Redistribution and use in source and binary forms, with or without
13 * modification, are permitted provided that the following conditions
14 * are met:
15 * 1. Redistributions of source code must retain the above copyright
16 * notice, this list of conditions and the following disclaimer.
17 * 2. Redistributions in binary form must reproduce the above copyright
18 * notice, this list of conditions and the following disclaimer in the
19 * documentation and/or other materials provided with the distribution.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
23 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
24 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
25 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
31 * POSSIBILITY OF SUCH DAMAGE.
32 */
33
34#include <sys/cdefs.h>
35__KERNEL_RCSID(0, "$NetBSD: lapic.c,v 1.53 2016/10/15 09:50:27 maxv Exp $");
36
37#include "opt_ddb.h"
38#include "opt_mpbios.h" /* for MPDEBUG */
39#include "opt_multiprocessor.h"
40#include "opt_ntp.h"
41
42#include <sys/param.h>
43#include <sys/proc.h>
44#include <sys/systm.h>
45#include <sys/device.h>
46#include <sys/timetc.h>
47
48#include <uvm/uvm_extern.h>
49
50#include <dev/ic/i8253reg.h>
51
52#include <machine/cpu.h>
53#include <machine/cpu_counter.h>
54#include <machine/cpufunc.h>
55#include <machine/cpuvar.h>
56#include <machine/pmap.h>
57#include <machine/vmparam.h>
58#include <machine/mpbiosvar.h>
59#include <machine/pcb.h>
60#include <machine/specialreg.h>
61#include <machine/segments.h>
62#include <x86/x86/tsc.h>
63#include <x86/i82093var.h>
64
65#include <machine/apicvar.h>
66#include <machine/i82489reg.h>
67#include <machine/i82489var.h>
68
69/* Referenced from vector.S */
70void lapic_clockintr(void *, struct intrframe *);
71
72static void lapic_delay(unsigned int);
73static uint32_t lapic_gettick(void);
74static void lapic_map(paddr_t);
75
76static void lapic_hwmask(struct pic *, int);
77static void lapic_hwunmask(struct pic *, int);
78static void lapic_setup(struct pic *, struct cpu_info *, int, int, int);
79/* Make it public to call via ddb */
80void lapic_dump(void);
81
82struct pic local_pic = {
83 .pic_name = "lapic",
84 .pic_type = PIC_LAPIC,
85 .pic_lock = __SIMPLELOCK_UNLOCKED,
86 .pic_hwmask = lapic_hwmask,
87 .pic_hwunmask = lapic_hwunmask,
88 .pic_addroute =lapic_setup,
89 .pic_delroute = lapic_setup,
90};
91
92static void
93lapic_map(paddr_t lapic_base)
94{
95 pt_entry_t *pte;
96 vaddr_t va = (vaddr_t)&local_apic;
97
98 /*
99 * If the CPU has an APIC MSR, use it and ignore the supplied value:
100 * some ACPI implementations have been observed to pass bad values.
101 * Additionally, ensure that the lapic is enabled as we are committed
102 * to using it at this point. Be conservative and assume that the MSR
103 * is not present on the Pentium (is it?).
104 */
105 if (CPUID_TO_FAMILY(curcpu()->ci_signature) >= 6) {
106 lapic_base = (paddr_t)rdmsr(LAPIC_MSR);
107 if ((lapic_base & LAPIC_MSR_ADDR) == 0) {
108 lapic_base |= LAPIC_BASE;
109 }
110 wrmsr(LAPIC_MSR, lapic_base | LAPIC_MSR_ENABLE);
111 lapic_base &= LAPIC_MSR_ADDR;
112 }
113
114 x86_disable_intr();
115
116 /*
117 * Map local apic. If we have a local apic, it's safe to assume
118 * we're on a 486 or better and can use invlpg and non-cacheable PTE's
119 *
120 * Whap the PTE "by hand" rather than calling pmap_kenter_pa because
121 * the latter will attempt to invoke TLB shootdown code just as we
122 * might have changed the value of cpu_number()..
123 */
124
125 pte = kvtopte(va);
126 *pte = lapic_base | PG_RW | PG_V | PG_N | pmap_pg_g | pmap_pg_nx;
127 invlpg(va);
128
129#ifdef MULTIPROCESSOR
130 cpu_init_first(); /* catch up to changed cpu_number() */
131#endif
132
133 i82489_writereg(LAPIC_TPRI, 0);
134 x86_enable_intr();
135}
136
137/*
138 * enable local apic
139 */
140void
141lapic_enable(void)
142{
143 i82489_writereg(LAPIC_SVR, LAPIC_SVR_ENABLE | LAPIC_SPURIOUS_VECTOR);
144}
145
146void
147lapic_set_lvt(void)
148{
149 struct cpu_info *ci = curcpu();
150 int i;
151 struct mp_intr_map *mpi;
152 uint32_t lint0, lint1;
153
154#ifdef MULTIPROCESSOR
155 if (mp_verbose) {
156 apic_format_redir(device_xname(ci->ci_dev), "prelint", 0, 0,
157 i82489_readreg(LAPIC_LVINT0));
158 apic_format_redir(device_xname(ci->ci_dev), "prelint", 1, 0,
159 i82489_readreg(LAPIC_LVINT1));
160 }
161#endif
162
163 /*
164 * If an I/O APIC has been attached, assume that it is used instead of
165 * the 8259A for interrupt delivery. Otherwise request the LAPIC to
166 * get external interrupts via LINT0 for the primary CPU.
167 */
168 lint0 = LAPIC_DLMODE_EXTINT;
169 if (nioapics > 0 || !CPU_IS_PRIMARY(curcpu()))
170 lint0 |= LAPIC_LVT_MASKED;
171 i82489_writereg(LAPIC_LVINT0, lint0);
172
173 /*
174 * Non Maskable Interrupts are to be delivered to the primary CPU.
175 */
176 lint1 = LAPIC_DLMODE_NMI;
177 if (!CPU_IS_PRIMARY(curcpu()))
178 lint1 |= LAPIC_LVT_MASKED;
179 i82489_writereg(LAPIC_LVINT1, lint1);
180
181 for (i = 0; i < mp_nintr; i++) {
182 mpi = &mp_intrs[i];
183 if (mpi->ioapic == NULL && (mpi->cpu_id == MPS_ALL_APICS ||
184 mpi->cpu_id == ci->ci_cpuid)) {
185 if (mpi->ioapic_pin > 1)
186 aprint_error_dev(ci->ci_dev,
187 "%s: WARNING: bad pin value %d\n",
188 __func__, mpi->ioapic_pin);
189 if (mpi->ioapic_pin == 0)
190 i82489_writereg(LAPIC_LVINT0, mpi->redir);
191 else
192 i82489_writereg(LAPIC_LVINT1, mpi->redir);
193 }
194 }
195
196#ifdef MULTIPROCESSOR
197 if (mp_verbose)
198 lapic_dump();
199#endif
200}
201
202/*
203 * Initialize fixed idt vectors for use by local apic.
204 */
205void
206lapic_boot_init(paddr_t lapic_base)
207{
208 lapic_map(lapic_base);
209
210#ifdef MULTIPROCESSOR
211 idt_vec_reserve(LAPIC_IPI_VECTOR);
212 idt_vec_set(LAPIC_IPI_VECTOR, Xintr_lapic_ipi);
213 idt_vec_reserve(LAPIC_TLB_VECTOR);
214 idt_vec_set(LAPIC_TLB_VECTOR, Xintr_lapic_tlb);
215#endif
216 idt_vec_reserve(LAPIC_SPURIOUS_VECTOR);
217 idt_vec_set(LAPIC_SPURIOUS_VECTOR, Xintrspurious);
218
219 idt_vec_reserve(LAPIC_TIMER_VECTOR);
220 idt_vec_set(LAPIC_TIMER_VECTOR, Xintr_lapic_ltimer);
221}
222
223static uint32_t
224lapic_gettick(void)
225{
226 return i82489_readreg(LAPIC_CCR_TIMER);
227}
228
229#include <sys/kernel.h> /* for hz */
230
231int lapic_timer = 0;
232uint32_t lapic_tval;
233
234/*
235 * this gets us up to a 4GHz busclock....
236 */
237uint32_t lapic_per_second;
238uint32_t lapic_frac_usec_per_cycle;
239uint64_t lapic_frac_cycle_per_usec;
240uint32_t lapic_delaytab[26];
241
242static u_int
243lapic_get_timecount(struct timecounter *tc)
244{
245 struct cpu_info *ci;
246 uint32_t cur_timer;
247 int s;
248
249 s = splhigh();
250 ci = curcpu();
251
252 /*
253 * Check for a race against the clockinterrupt.
254 * The update of ci_lapic_counter is blocked by splhigh() and
255 * the check for a pending clockinterrupt compensates for that.
256 *
257 * If the current tick is almost the Initial Counter, explicitly
258 * check for the pending interrupt bit as the interrupt delivery
259 * could be asynchronious and compensate as well.
260 *
261 * This can't be done without splhigh() as the calling code might
262 * have masked the clockinterrupt already.
263 *
264 * This code assumes that clockinterrupts are not missed.
265 */
266 cur_timer = lapic_gettick();
267 if (cur_timer >= lapic_tval - 1) {
268 uint16_t reg = LAPIC_IRR + LAPIC_TIMER_VECTOR / 32 * 16;
269
270 if (i82489_readreg(reg) & (1 << (LAPIC_TIMER_VECTOR % 32))) {
271 cur_timer -= lapic_tval;
272 }
273 } else if (ci->ci_istate.ipending & (1 << LIR_TIMER))
274 cur_timer = lapic_gettick() - lapic_tval;
275 cur_timer = ci->ci_lapic_counter - cur_timer;
276 splx(s);
277
278 return cur_timer;
279}
280
281static struct timecounter lapic_timecounter = {
282 lapic_get_timecount,
283 NULL,
284 ~0u,
285 0,
286 "lapic",
287#ifndef MULTIPROCESSOR
288 2100,
289#else
290 -100, /* per CPU state */
291#endif
292 NULL,
293 NULL,
294};
295
296extern u_int i8254_get_timecount(struct timecounter *);
297
298void
299lapic_clockintr(void *arg, struct intrframe *frame)
300{
301 struct cpu_info *ci = curcpu();
302
303 ci->ci_lapic_counter += lapic_tval;
304 ci->ci_isources[LIR_TIMER]->is_evcnt.ev_count++;
305 hardclock((struct clockframe *)frame);
306}
307
308void
309lapic_initclocks(void)
310{
311 /*
312 * Start local apic countdown timer running, in repeated mode.
313 *
314 * Mask the clock interrupt and set mode,
315 * then set divisor,
316 * then unmask and set the vector.
317 */
318 i82489_writereg(LAPIC_LVTT, LAPIC_LVTT_TM | LAPIC_LVTT_M);
319 i82489_writereg(LAPIC_DCR_TIMER, LAPIC_DCRT_DIV1);
320 i82489_writereg(LAPIC_ICR_TIMER, lapic_tval);
321 i82489_writereg(LAPIC_LVTT, LAPIC_LVTT_TM | LAPIC_TIMER_VECTOR);
322 i82489_writereg(LAPIC_EOI, 0);
323}
324
325extern unsigned int gettick(void); /* XXX put in header file */
326extern u_long rtclock_tval; /* XXX put in header file */
327extern void (*initclock_func)(void); /* XXX put in header file */
328
329/*
330 * Calibrate the local apic count-down timer (which is running at
331 * bus-clock speed) vs. the i8254 counter/timer (which is running at
332 * a fixed rate).
333 *
334 * The Intel MP spec says: "An MP operating system may use the IRQ8
335 * real-time clock as a reference to determine the actual APIC timer clock
336 * speed."
337 *
338 * We're actually using the IRQ0 timer. Hmm.
339 */
340void
341lapic_calibrate_timer(struct cpu_info *ci)
342{
343 unsigned int seen, delta, initial_i8254, initial_lapic;
344 unsigned int cur_i8254, cur_lapic;
345 uint64_t tmp;
346 int i;
347 char tbuf[9];
348
349 aprint_debug_dev(ci->ci_dev, "calibrating local timer\n");
350
351 /*
352 * Configure timer to one-shot, interrupt masked,
353 * large positive number.
354 */
355 i82489_writereg(LAPIC_LVTT, LAPIC_LVTT_M);
356 i82489_writereg(LAPIC_DCR_TIMER, LAPIC_DCRT_DIV1);
357 i82489_writereg(LAPIC_ICR_TIMER, 0x80000000);
358
359 x86_disable_intr();
360
361 initial_lapic = lapic_gettick();
362 initial_i8254 = gettick();
363
364 for (seen = 0; seen < TIMER_FREQ / 100; seen += delta) {
365 cur_i8254 = gettick();
366 if (cur_i8254 > initial_i8254)
367 delta = rtclock_tval - (cur_i8254 - initial_i8254);
368 else
369 delta = initial_i8254 - cur_i8254;
370 initial_i8254 = cur_i8254;
371 }
372 cur_lapic = lapic_gettick();
373
374 x86_enable_intr();
375
376 tmp = initial_lapic - cur_lapic;
377 lapic_per_second = (tmp * TIMER_FREQ + seen / 2) / seen;
378
379 humanize_number(tbuf, sizeof(tbuf), lapic_per_second, "Hz", 1000);
380
381 aprint_debug_dev(ci->ci_dev, "apic clock running at %s\n", tbuf);
382
383 if (lapic_per_second != 0) {
384 /*
385 * reprogram the apic timer to run in periodic mode.
386 * XXX need to program timer on other CPUs, too.
387 */
388 lapic_tval = (lapic_per_second * 2) / hz;
389 lapic_tval = (lapic_tval / 2) + (lapic_tval & 0x1);
390
391 i82489_writereg(LAPIC_LVTT, LAPIC_LVTT_TM | LAPIC_LVTT_M
392 | LAPIC_TIMER_VECTOR);
393 i82489_writereg(LAPIC_DCR_TIMER, LAPIC_DCRT_DIV1);
394 i82489_writereg(LAPIC_ICR_TIMER, lapic_tval);
395
396 /*
397 * Compute fixed-point ratios between cycles and
398 * microseconds to avoid having to do any division
399 * in lapic_delay.
400 */
401
402 tmp = (1000000 * (uint64_t)1 << 32) / lapic_per_second;
403 lapic_frac_usec_per_cycle = tmp;
404
405 tmp = (lapic_per_second * (uint64_t)1 << 32) / 1000000;
406
407 lapic_frac_cycle_per_usec = tmp;
408
409 /*
410 * Compute delay in cycles for likely short delays in usec.
411 */
412 for (i = 0; i < 26; i++)
413 lapic_delaytab[i] = (lapic_frac_cycle_per_usec * i) >>
414 32;
415
416 /*
417 * Now that the timer's calibrated, use the apic timer routines
418 * for all our timing needs..
419 */
420 delay_func = lapic_delay;
421 initclock_func = lapic_initclocks;
422 initrtclock(0);
423
424 if (lapic_timecounter.tc_frequency == 0) {
425 /*
426 * Hook up time counter.
427 * This assume that all LAPICs have the same frequency.
428 */
429 lapic_timecounter.tc_frequency = lapic_per_second;
430 tc_init(&lapic_timecounter);
431 }
432 }
433}
434
435/*
436 * delay for N usec.
437 */
438
439static void
440lapic_delay(unsigned int usec)
441{
442 int32_t xtick, otick;
443 int64_t deltat; /* XXX may want to be 64bit */
444
445 otick = lapic_gettick();
446
447 if (usec <= 0)
448 return;
449 if (usec <= 25)
450 deltat = lapic_delaytab[usec];
451 else
452 deltat = (lapic_frac_cycle_per_usec * usec) >> 32;
453
454 while (deltat > 0) {
455 xtick = lapic_gettick();
456 if (xtick > otick)
457 deltat -= lapic_tval - (xtick - otick);
458 else
459 deltat -= otick - xtick;
460 otick = xtick;
461
462 x86_pause();
463 }
464}
465
466/*
467 * XXX the following belong mostly or partly elsewhere..
468 */
469
470static void
471i82489_icr_wait(void)
472{
473#ifdef DIAGNOSTIC
474 unsigned j = 100000;
475#endif /* DIAGNOSTIC */
476
477 while ((i82489_readreg(LAPIC_ICRLO) & LAPIC_DLSTAT_BUSY) != 0) {
478 x86_pause();
479#ifdef DIAGNOSTIC
480 j--;
481 if (j == 0)
482 panic("i82489_icr_wait: busy");
483#endif /* DIAGNOSTIC */
484 }
485}
486
487int
488x86_ipi_init(int target)
489{
490 uint32_t esr;
491
492 i82489_writereg(LAPIC_ESR, 0);
493 (void)i82489_readreg(LAPIC_ESR);
494
495 if ((target&LAPIC_DEST_MASK)==0)
496 i82489_writereg(LAPIC_ICRHI, target<<LAPIC_ID_SHIFT);
497
498 i82489_writereg(LAPIC_ICRLO, (target & LAPIC_DEST_MASK) |
499 LAPIC_DLMODE_INIT | LAPIC_LEVEL_ASSERT );
500 i82489_icr_wait();
501 i8254_delay(10000);
502 i82489_writereg(LAPIC_ICRLO, (target & LAPIC_DEST_MASK) |
503 LAPIC_DLMODE_INIT | LAPIC_TRIGGER_LEVEL | LAPIC_LEVEL_DEASSERT);
504 i82489_icr_wait();
505
506 if ((i82489_readreg(LAPIC_ICRLO) & LAPIC_DLSTAT_BUSY) != 0)
507 return EBUSY;
508
509 esr = i82489_readreg(LAPIC_ESR);
510 if (esr != 0)
511 aprint_debug("x86_ipi_init: ESR %08x\n", esr);
512
513 return 0;
514}
515
516int
517x86_ipi_startup(int target, int vec)
518{
519 uint32_t esr;
520
521 i82489_writereg(LAPIC_ESR, 0);
522 (void)i82489_readreg(LAPIC_ESR);
523
524 i82489_icr_wait();
525 i82489_writereg(LAPIC_ICRHI, target << LAPIC_ID_SHIFT);
526 i82489_writereg(LAPIC_ICRLO, vec | LAPIC_DLMODE_STARTUP |
527 LAPIC_LEVEL_ASSERT);
528 i82489_icr_wait();
529
530 if ((i82489_readreg(LAPIC_ICRLO) & LAPIC_DLSTAT_BUSY) != 0)
531 return EBUSY;
532
533 esr = i82489_readreg(LAPIC_ESR);
534 if (esr != 0)
535 aprint_debug("x86_ipi_startup: ESR %08x\n", esr);
536
537 return 0;
538}
539
540int
541x86_ipi(int vec, int target, int dl)
542{
543 int result, s;
544
545 s = splhigh();
546
547 i82489_icr_wait();
548
549 if ((target & LAPIC_DEST_MASK) == 0)
550 i82489_writereg(LAPIC_ICRHI, target << LAPIC_ID_SHIFT);
551
552 i82489_writereg(LAPIC_ICRLO,
553 (target & LAPIC_DEST_MASK) | vec | dl | LAPIC_LEVEL_ASSERT);
554
555#ifdef DIAGNOSTIC
556 i82489_icr_wait();
557 result = (i82489_readreg(LAPIC_ICRLO) & LAPIC_DLSTAT_BUSY) ? EBUSY : 0;
558#else
559 /* Don't wait - if it doesn't go, we're in big trouble anyway. */
560 result = 0;
561#endif
562 splx(s);
563
564 return result;
565}
566
567
568/*
569 * Using 'pin numbers' as:
570 * 0 - timer
571 * 1 - unused
572 * 2 - PCINT
573 * 3 - LVINT0
574 * 4 - LVINT1
575 * 5 - LVERR
576 */
577
578static void
579lapic_hwmask(struct pic *pic, int pin)
580{
581 int reg;
582 uint32_t val;
583
584 reg = LAPIC_LVTT + (pin << 4);
585 val = i82489_readreg(reg);
586 val |= LAPIC_LVT_MASKED;
587 i82489_writereg(reg, val);
588}
589
590static void
591lapic_hwunmask(struct pic *pic, int pin)
592{
593 int reg;
594 uint32_t val;
595
596 reg = LAPIC_LVTT + (pin << 4);
597 val = i82489_readreg(reg);
598 val &= ~LAPIC_LVT_MASKED;
599 i82489_writereg(reg, val);
600}
601
602static void
603lapic_setup(struct pic *pic, struct cpu_info *ci,
604 int pin, int idtvec, int type)
605{
606}
607
608void
609lapic_dump(void)
610{
611 struct cpu_info *ci = curcpu();
612
613 apic_format_redir(device_xname(ci->ci_dev), "timer", 0, 0,
614 i82489_readreg(LAPIC_LVTT));
615 apic_format_redir(device_xname(ci->ci_dev), "pcint", 0, 0,
616 i82489_readreg(LAPIC_PCINT));
617 apic_format_redir(device_xname(ci->ci_dev), "lint", 0, 0,
618 i82489_readreg(LAPIC_LVINT0));
619 apic_format_redir(device_xname(ci->ci_dev), "lint", 1, 0,
620 i82489_readreg(LAPIC_LVINT1));
621 apic_format_redir(device_xname(ci->ci_dev), "err", 0, 0,
622 i82489_readreg(LAPIC_LVERR));
623}
624