1/* $NetBSD: pci_msi_machdep.c,v 1.9 2015/08/17 06:16:03 knakahara Exp $ */
2
3/*
4 * Copyright (c) 2015 Internet Initiative Japan Inc.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
17 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
18 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26 * POSSIBILITY OF SUCH DAMAGE.
27 */
28
29/*
30 * TODO
31 *
32 * - PBA (Pending Bit Array) support
33 * - HyperTransport mapping support
34 */
35
36#include <sys/cdefs.h>
37__KERNEL_RCSID(0, "$NetBSD: pci_msi_machdep.c,v 1.9 2015/08/17 06:16:03 knakahara Exp $");
38
39#include "opt_intrdebug.h"
40
41#include <sys/types.h>
42#include <sys/param.h>
43#include <sys/time.h>
44#include <sys/systm.h>
45#include <sys/cpu.h>
46#include <sys/errno.h>
47#include <sys/device.h>
48#include <sys/intr.h>
49#include <sys/kmem.h>
50#include <sys/malloc.h>
51
52#include <dev/pci/pcivar.h>
53
54#include <machine/i82093var.h>
55#include <machine/pic.h>
56
57#include <x86/pci/msipic.h>
58#include <x86/pci/pci_msi_machdep.h>
59
60#ifdef INTRDEBUG
61#define MSIDEBUG
62#endif
63
64#ifdef MSIDEBUG
65#define DPRINTF(msg) printf msg
66#else
67#define DPRINTF(msg)
68#endif
69
70static pci_intr_handle_t
71pci_msi_calculate_handle(struct pic *msi_pic, int vector)
72{
73 pci_intr_handle_t pih;
74
75 KASSERT(msipic_is_msi_pic(msi_pic));
76
77 pih = __SHIFTIN((uint64_t)msipic_get_devid(msi_pic), MSI_INT_DEV_MASK)
78 | __SHIFTIN((uint64_t)vector, MSI_INT_VEC_MASK)
79 | APIC_INT_VIA_MSI;
80 if (msi_pic->pic_type == PIC_MSI)
81 MSI_INT_MAKE_MSI(pih);
82 else if (msi_pic->pic_type == PIC_MSIX)
83 MSI_INT_MAKE_MSIX(pih);
84 else
85 panic("%s: Unexpected pic_type: %d\n", __func__,
86 msi_pic->pic_type);
87
88 return pih;
89}
90
91static pci_intr_handle_t *
92pci_msi_alloc_vectors(struct pic *msi_pic, uint *table_indexes, int *count)
93{
94 struct intrsource *isp;
95 pci_intr_handle_t *vectors, pih;
96 int i;
97 const char *intrstr;
98 char intrstr_buf[INTRIDBUF];
99
100 vectors = kmem_zalloc(sizeof(vectors[0]) * (*count), KM_SLEEP);
101 if (vectors == NULL) {
102 DPRINTF(("cannot allocate vectors\n"));
103 return NULL;
104 }
105
106 mutex_enter(&cpu_lock);
107 for (i = 0; i < *count; i++) {
108 u_int table_index;
109
110 if (table_indexes == NULL)
111 table_index = i;
112 else
113 table_index = table_indexes[i];
114
115 pih = pci_msi_calculate_handle(msi_pic, table_index);
116
117 intrstr = x86_pci_msi_string(NULL, pih, intrstr_buf,
118 sizeof(intrstr_buf));
119 isp = intr_allocate_io_intrsource(intrstr);
120 if (isp == NULL) {
121 mutex_exit(&cpu_lock);
122 DPRINTF(("can't allocate io_intersource\n"));
123 kmem_free(vectors, sizeof(vectors[0]) * (*count));
124 return NULL;
125 }
126
127 vectors[i] = pih;
128 }
129 mutex_exit(&cpu_lock);
130
131 return vectors;
132}
133
134static void
135pci_msi_free_vectors(struct pic *msi_pic, pci_intr_handle_t *pihs, int count)
136{
137 pci_intr_handle_t pih;
138 int i;
139 const char *intrstr;
140 char intrstr_buf[INTRIDBUF];
141
142 mutex_enter(&cpu_lock);
143 for (i = 0; i < count; i++) {
144 pih = pci_msi_calculate_handle(msi_pic, i);
145 intrstr = x86_pci_msi_string(NULL, pih, intrstr_buf,
146 sizeof(intrstr_buf));
147 intr_free_io_intrsource(intrstr);
148 }
149 mutex_exit(&cpu_lock);
150
151 kmem_free(pihs, sizeof(pihs[0]) * count);
152}
153
154static int
155pci_msi_alloc_common(pci_intr_handle_t **ihps, int *count,
156 const struct pci_attach_args *pa, bool exact)
157{
158 struct pic *msi_pic;
159 pci_intr_handle_t *vectors;
160 int error, i;
161
162 if ((pa->pa_flags & PCI_FLAGS_MSI_OKAY) == 0) {
163 DPRINTF(("PCI host bridge does not support MSI.\n"));
164 return ENODEV;
165 }
166
167 msi_pic = msipic_construct_msi_pic(pa);
168 if (msi_pic == NULL) {
169 DPRINTF(("cannot allocate MSI pic.\n"));
170 return EINVAL;
171 }
172
173 vectors = NULL;
174 while (*count > 0) {
175 vectors = pci_msi_alloc_vectors(msi_pic, NULL, count);
176 if (vectors != NULL)
177 break;
178
179 if (exact) {
180 DPRINTF(("cannot allocate MSI vectors.\n"));
181 msipic_destruct_msi_pic(msi_pic);
182 return ENOMEM;
183 } else {
184 (*count) >>= 1; /* must be power of 2. */
185 continue;
186 }
187 }
188 if (vectors == NULL) {
189 DPRINTF(("cannot allocate MSI vectors.\n"));
190 msipic_destruct_msi_pic(msi_pic);
191 return ENOMEM;
192 }
193
194 for (i = 0; i < *count; i++) {
195 MSI_INT_MAKE_MSI(vectors[i]);
196 }
197
198 error = msipic_set_msi_vectors(msi_pic, NULL, *count);
199 if (error) {
200 pci_msi_free_vectors(msi_pic, vectors, *count);
201 msipic_destruct_msi_pic(msi_pic);
202 return error;
203 }
204
205 *ihps = vectors;
206 return 0;
207}
208
209static void *
210pci_msi_common_establish(pci_chipset_tag_t pc, pci_intr_handle_t ih,
211 int level, int (*func)(void *), void *arg, struct pic *pic,
212 const char *xname)
213{
214 int irq, pin;
215 bool mpsafe;
216
217 KASSERT(INT_VIA_MSI(ih));
218
219 irq = -1;
220 pin = MSI_INT_VEC(ih);
221 mpsafe = ((ih & MPSAFE_MASK) != 0);
222
223 return intr_establish_xname(irq, pic, pin, IST_EDGE, level, func, arg,
224 mpsafe, xname);
225}
226
227static void
228pci_msi_common_disestablish(pci_chipset_tag_t pc, void *cookie)
229{
230
231 intr_disestablish(cookie);
232}
233
234static int
235pci_msix_alloc_common(pci_intr_handle_t **ihps, u_int *table_indexes,
236 int *count, const struct pci_attach_args *pa, bool exact)
237{
238 struct pic *msix_pic;
239 pci_intr_handle_t *vectors;
240 int error, i;
241
242 if ((pa->pa_flags & PCI_FLAGS_MSIX_OKAY) == 0) {
243 DPRINTF(("PCI host bridge does not support MSI-X.\n"));
244 return ENODEV;
245 }
246
247 msix_pic = msipic_construct_msix_pic(pa);
248 if (msix_pic == NULL)
249 return EINVAL;
250
251 vectors = NULL;
252 while (*count > 0) {
253 vectors = pci_msi_alloc_vectors(msix_pic, table_indexes, count);
254 if (vectors != NULL)
255 break;
256
257 if (exact) {
258 DPRINTF(("cannot allocate MSI-X vectors.\n"));
259 msipic_destruct_msix_pic(msix_pic);
260 return ENOMEM;
261 } else {
262 (*count)--;
263 continue;
264 }
265 }
266 if (vectors == NULL) {
267 DPRINTF(("cannot allocate MSI-X vectors.\n"));
268 msipic_destruct_msix_pic(msix_pic);
269 return ENOMEM;
270 }
271
272 for (i = 0; i < *count; i++) {
273 MSI_INT_MAKE_MSIX(vectors[i]);
274 }
275
276 error = msipic_set_msi_vectors(msix_pic, vectors, *count);
277 if (error) {
278 pci_msi_free_vectors(msix_pic, vectors, *count);
279 msipic_destruct_msix_pic(msix_pic);
280 return error;
281 }
282
283 *ihps = vectors;
284 return 0;
285}
286
287static int
288x86_pci_msi_alloc(pci_intr_handle_t **ihps, int *count,
289 const struct pci_attach_args *pa)
290{
291
292 return pci_msi_alloc_common(ihps, count, pa, false);
293}
294
295static int
296x86_pci_msi_alloc_exact(pci_intr_handle_t **ihps, int count,
297 const struct pci_attach_args *pa)
298{
299
300 return pci_msi_alloc_common(ihps, &count, pa, true);
301}
302
303static void
304x86_pci_msi_release_internal(pci_intr_handle_t *pihs, int count)
305{
306 struct pic *pic;
307
308 pic = msipic_find_msi_pic(MSI_INT_DEV(pihs[0]));
309 if (pic == NULL)
310 return;
311
312 pci_msi_free_vectors(pic, pihs, count);
313 msipic_destruct_msi_pic(pic);
314}
315
316static int
317x86_pci_msix_alloc(pci_intr_handle_t **ihps, int *count,
318 const struct pci_attach_args *pa)
319{
320
321 return pci_msix_alloc_common(ihps, NULL, count, pa, false);
322}
323
324static int
325x86_pci_msix_alloc_exact(pci_intr_handle_t **ihps, int count,
326 const struct pci_attach_args *pa)
327{
328
329 return pci_msix_alloc_common(ihps, NULL, &count, pa, true);
330}
331
332static int
333x86_pci_msix_alloc_map(pci_intr_handle_t **ihps, u_int *table_indexes,
334 int count, const struct pci_attach_args *pa)
335{
336
337 return pci_msix_alloc_common(ihps, table_indexes, &count, pa, true);
338}
339
340static void
341x86_pci_msix_release_internal(pci_intr_handle_t *pihs, int count)
342{
343 struct pic *pic;
344
345 pic = msipic_find_msi_pic(MSI_INT_DEV(pihs[0]));
346 if (pic == NULL)
347 return;
348
349 pci_msi_free_vectors(pic, pihs, count);
350 msipic_destruct_msix_pic(pic);
351}
352
353/*****************************************************************************/
354/*
355 * extern for MD code.
356 */
357
358/*
359 * Return intrid for a MSI/MSI-X device.
360 * "buf" must be allocated by caller.
361 */
362const char *
363x86_pci_msi_string(pci_chipset_tag_t pc, pci_intr_handle_t ih, char *buf,
364 size_t len)
365{
366 int dev, vec;
367
368 KASSERT(INT_VIA_MSI(ih));
369
370 dev = MSI_INT_DEV(ih);
371 vec = MSI_INT_VEC(ih);
372 if (MSI_INT_IS_MSIX(ih))
373 snprintf(buf, len, "msix%d vec %d", dev, vec);
374 else
375 snprintf(buf, len, "msi%d vec %d", dev, vec);
376
377 return buf;
378}
379
380/*
381 * Release MSI handles.
382 */
383void
384x86_pci_msi_release(pci_chipset_tag_t pc, pci_intr_handle_t *pihs, int count)
385{
386
387 if (count < 1)
388 return;
389
390 return x86_pci_msi_release_internal(pihs, count);
391}
392
393/*
394 * Establish a MSI handle.
395 * If multiple MSI handle is requied to establish, device driver must call
396 * this function for each handle.
397 */
398void *
399x86_pci_msi_establish(pci_chipset_tag_t pc, pci_intr_handle_t ih,
400 int level, int (*func)(void *), void *arg, const char *xname)
401{
402 struct pic *pic;
403
404 pic = msipic_find_msi_pic(MSI_INT_DEV(ih));
405 if (pic == NULL) {
406 DPRINTF(("pci_intr_handler has no msi_pic\n"));
407 return NULL;
408 }
409
410 return pci_msi_common_establish(pc, ih, level, func, arg, pic, xname);
411}
412
413/*
414 * Disestablish a MSI handle.
415 * If multiple MSI handle is requied to disestablish, device driver must call
416 * this function for each handle.
417 */
418void
419x86_pci_msi_disestablish(pci_chipset_tag_t pc, void *cookie)
420{
421
422 pci_msi_common_disestablish(pc, cookie);
423}
424
425/*
426 * Release MSI-X handles.
427 */
428void
429x86_pci_msix_release(pci_chipset_tag_t pc, pci_intr_handle_t *pihs, int count)
430{
431
432 if (count < 1)
433 return;
434
435 return x86_pci_msix_release_internal(pihs, count);
436}
437
438/*
439 * Establish a MSI-X handle.
440 * If multiple MSI-X handle is requied to establish, device driver must call
441 * this function for each handle.
442 */
443void *
444x86_pci_msix_establish(pci_chipset_tag_t pc, pci_intr_handle_t ih,
445 int level, int (*func)(void *), void *arg, const char *xname)
446{
447 struct pic *pic;
448
449 pic = msipic_find_msi_pic(MSI_INT_DEV(ih));
450 if (pic == NULL) {
451 DPRINTF(("pci_intr_handler has no msi_pic\n"));
452 return NULL;
453 }
454
455 return pci_msi_common_establish(pc, ih, level, func, arg, pic, xname);
456}
457
458/*
459 * Disestablish a MSI-X handle.
460 * If multiple MSI-X handle is requied to disestablish, device driver must call
461 * this function for each handle.
462 */
463void
464x86_pci_msix_disestablish(pci_chipset_tag_t pc, void *cookie)
465{
466
467 pci_msi_common_disestablish(pc, cookie);
468}
469
470/*****************************************************************************/
471/*
472 * extern for MI code.
473 */
474
475/*
476 * This function is used by device drivers like pci_intr_map().
477 *
478 * "ihps" is the array of vector numbers which MSI used instead of IRQ number.
479 * "count" must be power of 2.
480 * "count" can decrease if struct intrsource cannot be allocated.
481 * if count == 0, return non-zero value.
482 */
483int
484pci_msi_alloc(const struct pci_attach_args *pa, pci_intr_handle_t **ihps,
485 int *count)
486{
487 int hw_max;
488
489 /* MSI vector count must be power of 2. */
490 KASSERT(*count > 0);
491 KASSERT(((*count - 1) & *count) == 0);
492
493 hw_max = pci_msi_count(pa->pa_pc, pa->pa_tag);
494 if (hw_max == 0)
495 return ENODEV;
496
497 if (*count > hw_max) {
498 DPRINTF(("cut off MSI count to %d\n", hw_max));
499 *count = hw_max; /* cut off hw_max */
500 }
501
502 return x86_pci_msi_alloc(ihps, count, pa);
503}
504
505/*
506 * This function is used by device drivers like pci_intr_map().
507 *
508 * "ihps" is the array of vector numbers which MSI used instead of IRQ number.
509 * "count" must be power of 2.
510 * "count" can not decrease.
511 * If "count" struct intrsources cannot be allocated, return non-zero value.
512 */
513int
514pci_msi_alloc_exact(const struct pci_attach_args *pa, pci_intr_handle_t **ihps,
515 int count)
516{
517 int hw_max;
518
519 /* MSI vector count must be power of 2. */
520 KASSERT(count > 0);
521 KASSERT(((count - 1) & count) == 0);
522
523 hw_max = pci_msi_count(pa->pa_pc, pa->pa_tag);
524 if (hw_max == 0)
525 return ENODEV;
526
527 if (count > hw_max) {
528 DPRINTF(("over hardware max MSI count %d\n", hw_max));
529 return EINVAL;
530 }
531
532 return x86_pci_msi_alloc_exact(ihps, count, pa);
533}
534
535/*
536 * This function is used by device drivers like pci_intr_map().
537 *
538 * "ihps" is the array of vector numbers which MSI-X used instead of IRQ number.
539 * "count" can decrease if enough struct intrsources cannot be allocated.
540 * if count == 0, return non-zero value.
541 */
542int
543pci_msix_alloc(const struct pci_attach_args *pa, pci_intr_handle_t **ihps,
544 int *count)
545{
546 int hw_max;
547
548 KASSERT(*count > 0);
549
550 hw_max = pci_msix_count(pa->pa_pc, pa->pa_tag);
551 if (hw_max == 0)
552 return ENODEV;
553
554 if (*count > hw_max) {
555 DPRINTF(("cut off MSI-X count to %d\n", hw_max));
556 *count = hw_max; /* cut off hw_max */
557 }
558
559 return x86_pci_msix_alloc(ihps, count, pa);
560}
561
562/*
563 * This function is used by device drivers like pci_intr_map().
564 *
565 * "ihps" is the array of vector numbers which MSI-X used instead of IRQ number.
566 * "count" can not decrease.
567 * If "count" struct intrsource cannot be allocated, return non-zero value.
568 */
569int
570pci_msix_alloc_exact(const struct pci_attach_args *pa, pci_intr_handle_t **ihps,
571 int count)
572{
573 int hw_max;
574
575 KASSERT(count > 0);
576
577 hw_max = pci_msix_count(pa->pa_pc, pa->pa_tag);
578 if (hw_max == 0)
579 return ENODEV;
580
581 if (count > hw_max) {
582 DPRINTF(("over hardware max MSI-X count %d\n", hw_max));
583 return EINVAL;
584 }
585
586 return x86_pci_msix_alloc_exact(ihps, count, pa);
587}
588
589/*
590 * This function is used by device drivers like pci_intr_map().
591 * Futhermore, this function can map each handle to a MSI-X table index.
592 *
593 * "ihps" is the array of vector numbers which MSI-X used instead of IRQ number.
594 * "count" can not decrease.
595 * "map" size must be equal to "count".
596 * If "count" struct intrsource cannot be allocated, return non-zero value.
597 * e.g.
598 * If "map" = { 1, 4, 0 },
599 * 1st handle is bound to MSI-X index 1
600 * 2nd handle is bound to MSI-X index 4
601 * 3rd handle is bound to MSI-X index 0
602 */
603int
604pci_msix_alloc_map(const struct pci_attach_args *pa, pci_intr_handle_t **ihps,
605 u_int *table_indexes, int count)
606{
607 int hw_max, i, j;
608
609 KASSERT(count > 0);
610
611 hw_max = pci_msix_count(pa->pa_pc, pa->pa_tag);
612 if (hw_max == 0)
613 return ENODEV;
614
615 if (count > hw_max) {
616 DPRINTF(("over hardware max MSI-X count %d\n", hw_max));
617 return EINVAL;
618 }
619
620 /* check not to duplicate table_index */
621 for (i = 0; i < count; i++) {
622 u_int basing = table_indexes[i];
623
624 KASSERT(table_indexes[i] < PCI_MSIX_MAX_VECTORS);
625 if (basing >= hw_max) {
626 DPRINTF(("table index is over hardware max MSI-X index %d\n",
627 hw_max - 1));
628 return EINVAL;
629 }
630
631 for (j = i + 1; j < count; j++) {
632 if (basing == table_indexes[j]) {
633 DPRINTF(("MSI-X table index duplicated\n"));
634 return EINVAL;
635 }
636 }
637 }
638
639 return x86_pci_msix_alloc_map(ihps, table_indexes, count, pa);
640}
641