1 | /* $NetBSD: msipic.c,v 1.8 2015/11/17 17:51:42 msaitoh Exp $ */ |
2 | |
3 | /* |
4 | * Copyright (c) 2015 Internet Initiative Japan Inc. |
5 | * All rights reserved. |
6 | * |
7 | * Redistribution and use in source and binary forms, with or without |
8 | * modification, are permitted provided that the following conditions |
9 | * are met: |
10 | * 1. Redistributions of source code must retain the above copyright |
11 | * notice, this list of conditions and the following disclaimer. |
12 | * 2. Redistributions in binary form must reproduce the above copyright |
13 | * notice, this list of conditions and the following disclaimer in the |
14 | * documentation and/or other materials provided with the distribution. |
15 | * |
16 | * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS |
17 | * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED |
18 | * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
19 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS |
20 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
21 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
22 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS |
23 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN |
24 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
25 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
26 | * POSSIBILITY OF SUCH DAMAGE. |
27 | */ |
28 | |
29 | #include <sys/cdefs.h> |
30 | __KERNEL_RCSID(0, "$NetBSD: msipic.c,v 1.8 2015/11/17 17:51:42 msaitoh Exp $" ); |
31 | |
32 | #include "opt_intrdebug.h" |
33 | |
34 | #include <sys/types.h> |
35 | #include <sys/param.h> |
36 | #include <sys/systm.h> |
37 | #include <sys/errno.h> |
38 | #include <sys/kmem.h> |
39 | #include <sys/malloc.h> |
40 | #include <sys/mutex.h> |
41 | |
42 | #include <dev/pci/pcivar.h> |
43 | |
44 | #include <machine/i82489reg.h> |
45 | #include <machine/i82093reg.h> |
46 | #include <machine/i82093var.h> |
47 | #include <machine/pic.h> |
48 | #include <machine/lock.h> |
49 | |
50 | #include <x86/pci/msipic.h> |
51 | |
52 | #ifdef INTRDEBUG |
53 | #define MSIPICDEBUG |
54 | #endif |
55 | |
56 | #ifdef MSIPICDEBUG |
57 | #define DPRINTF(msg) printf msg |
58 | #else |
59 | #define DPRINTF(msg) |
60 | #endif |
61 | |
62 | #define BUS_SPACE_WRITE_FLUSH(pc, tag) (void)bus_space_read_4(pc, tag, 0) |
63 | |
64 | #define MSIPICNAMEBUF 16 |
65 | |
66 | /* |
67 | * A Pseudo pic for single MSI/MSI-X device. |
68 | * The pic and MSI/MSI-X device are distinbuished by "devid". The "devid" |
69 | * is managed by below "dev_seqs". |
70 | */ |
71 | struct msipic { |
72 | int mp_bus; |
73 | int mp_dev; |
74 | int mp_fun; |
75 | |
76 | int mp_devid; /* The device id for the MSI/MSI-X device. */ |
77 | int mp_veccnt; /* The number of MSI/MSI-X vectors. */ |
78 | |
79 | char mp_pic_name[MSIPICNAMEBUF]; /* The MSI/MSI-X device's name. */ |
80 | |
81 | struct pci_attach_args mp_pa; |
82 | bus_space_tag_t mp_bstag; |
83 | bus_space_handle_t mp_bshandle; |
84 | bus_size_t mp_bssize; |
85 | struct pic *mp_pic; |
86 | |
87 | LIST_ENTRY(msipic) mp_list; |
88 | }; |
89 | |
90 | static kmutex_t msipic_list_lock; |
91 | |
92 | static LIST_HEAD(, msipic) msipic_list = |
93 | LIST_HEAD_INITIALIZER(msipic_list); |
94 | |
95 | /* |
96 | * This struct managements "devid" to use the same "devid" for the device |
97 | * re-attached. If the device's bus number and device numer and function |
98 | * number are equal, it is assumed re-attached. |
99 | */ |
100 | struct dev_last_used_seq { |
101 | bool ds_using; |
102 | int ds_bus; |
103 | int ds_dev; |
104 | int ds_fun; |
105 | }; |
106 | /* The number of MSI/MSI-X devices supported by system. */ |
107 | #define NUM_MSI_DEVS 256 |
108 | /* Record devids to use the same devid when the device is re-attached. */ |
109 | static struct dev_last_used_seq dev_seqs[NUM_MSI_DEVS]; |
110 | |
111 | static int msipic_allocate_common_msi_devid(const struct pci_attach_args *); |
112 | static void msipic_release_common_msi_devid(int); |
113 | |
114 | static struct pic *msipic_find_msi_pic_locked(int); |
115 | static struct pic *msipic_construct_common_msi_pic(const struct pci_attach_args *, |
116 | struct pic *); |
117 | static void msipic_destruct_common_msi_pic(struct pic *); |
118 | |
119 | static void msi_set_msictl_enablebit(struct pic *, int, int); |
120 | static void msi_hwmask(struct pic *, int); |
121 | static void msi_hwunmask(struct pic *, int); |
122 | static void msi_addroute(struct pic *, struct cpu_info *, int, int, int); |
123 | static void msi_delroute(struct pic *, struct cpu_info *, int, int, int); |
124 | |
125 | static void msix_set_vecctl_mask(struct pic *, int, int); |
126 | static void msix_hwmask(struct pic *, int); |
127 | static void msix_hwunmask(struct pic *, int); |
128 | static void msix_addroute(struct pic *, struct cpu_info *, int, int, int); |
129 | static void msix_delroute(struct pic *, struct cpu_info *, int, int, int); |
130 | |
131 | /* |
132 | * Return new "devid" for the device attached first. |
133 | * Return the same "devid" for the device re-attached after dettached once. |
134 | * Return -1 if the number of attached MSI/MSI-X devices is over NUM_MSI_DEVS. |
135 | */ |
136 | static int |
137 | msipic_allocate_common_msi_devid(const struct pci_attach_args *pa) |
138 | { |
139 | pci_chipset_tag_t pc; |
140 | pcitag_t tag; |
141 | int bus, dev, fun, i; |
142 | |
143 | KASSERT(mutex_owned(&msipic_list_lock)); |
144 | |
145 | pc = pa->pa_pc; |
146 | tag = pa->pa_tag; |
147 | pci_decompose_tag(pc, tag, &bus, &dev, &fun); |
148 | |
149 | /* if the device was once attached, use same devid */ |
150 | for (i = 0; i < NUM_MSI_DEVS; i++) { |
151 | /* skip host bridge */ |
152 | if (dev_seqs[i].ds_bus == 0 |
153 | && dev_seqs[i].ds_dev == 0 |
154 | && dev_seqs[i].ds_fun == 0) |
155 | break; |
156 | |
157 | if (dev_seqs[i].ds_bus == bus |
158 | && dev_seqs[i].ds_dev == dev |
159 | && dev_seqs[i].ds_fun == fun) { |
160 | dev_seqs[i].ds_using = true; |
161 | return i; |
162 | } |
163 | } |
164 | |
165 | for (i = 0; i < NUM_MSI_DEVS; i++) { |
166 | if (dev_seqs[i].ds_using == 0) { |
167 | dev_seqs[i].ds_using = true; |
168 | dev_seqs[i].ds_bus = bus; |
169 | dev_seqs[i].ds_dev = dev; |
170 | dev_seqs[i].ds_fun = fun; |
171 | return i; |
172 | } |
173 | } |
174 | |
175 | DPRINTF(("too many MSI devices.\n" )); |
176 | return -1; |
177 | } |
178 | |
179 | /* |
180 | * Set the "devid" unused, but keep reserving the "devid" to reuse when |
181 | * the device is re-attached. |
182 | */ |
183 | static void |
184 | msipic_release_common_msi_devid(int devid) |
185 | { |
186 | |
187 | KASSERT(mutex_owned(&msipic_list_lock)); |
188 | |
189 | if (devid < 0 || NUM_MSI_DEVS <= devid) { |
190 | DPRINTF(("%s: invalid devid.\n" , __func__)); |
191 | return; |
192 | } |
193 | |
194 | dev_seqs[devid].ds_using = false; |
195 | /* Keep ds_* to reuse the same devid for the same device. */ |
196 | } |
197 | |
198 | static struct pic * |
199 | msipic_find_msi_pic_locked(int devid) |
200 | { |
201 | struct msipic *mpp; |
202 | |
203 | KASSERT(mutex_owned(&msipic_list_lock)); |
204 | |
205 | LIST_FOREACH(mpp, &msipic_list, mp_list) { |
206 | if(mpp->mp_devid == devid) |
207 | return mpp->mp_pic; |
208 | } |
209 | return NULL; |
210 | } |
211 | |
212 | /* |
213 | * Return the msi_pic whose device is already registered. |
214 | * If the device is not registered yet, return NULL. |
215 | */ |
216 | struct pic * |
217 | msipic_find_msi_pic(int devid) |
218 | { |
219 | struct pic *msipic; |
220 | |
221 | mutex_enter(&msipic_list_lock); |
222 | msipic = msipic_find_msi_pic_locked(devid); |
223 | mutex_exit(&msipic_list_lock); |
224 | |
225 | return msipic; |
226 | } |
227 | |
228 | /* |
229 | * A common construct process of MSI and MSI-X. |
230 | */ |
231 | static struct pic * |
232 | msipic_construct_common_msi_pic(const struct pci_attach_args *pa, |
233 | struct pic *pic_tmpl) |
234 | { |
235 | struct pic *pic; |
236 | struct msipic *msipic; |
237 | int devid; |
238 | |
239 | pic = kmem_alloc(sizeof(*pic), KM_SLEEP); |
240 | if (pic == NULL) |
241 | return NULL; |
242 | |
243 | msipic = kmem_zalloc(sizeof(*msipic), KM_SLEEP); |
244 | if (msipic == NULL) { |
245 | kmem_free(pic, sizeof(*pic)); |
246 | return NULL; |
247 | } |
248 | |
249 | mutex_enter(&msipic_list_lock); |
250 | |
251 | devid = msipic_allocate_common_msi_devid(pa); |
252 | if (devid == -1) { |
253 | mutex_exit(&msipic_list_lock); |
254 | kmem_free(pic, sizeof(*pic)); |
255 | kmem_free(msipic, sizeof(*msipic)); |
256 | return NULL; |
257 | } |
258 | |
259 | memcpy(pic, pic_tmpl, sizeof(*pic)); |
260 | pic->pic_msipic = msipic; |
261 | msipic->mp_pic = pic; |
262 | pci_decompose_tag(pa->pa_pc, pa->pa_tag, |
263 | &msipic->mp_bus, &msipic->mp_dev, &msipic->mp_fun); |
264 | memcpy(&msipic->mp_pa, pa, sizeof(msipic->mp_pa)); |
265 | msipic->mp_devid = devid; |
266 | /* |
267 | * pci_msi{,x}_alloc() must be called only once in the device driver. |
268 | */ |
269 | KASSERT(msipic_find_msi_pic_locked(msipic->mp_devid) == NULL); |
270 | |
271 | LIST_INSERT_HEAD(&msipic_list, msipic, mp_list); |
272 | |
273 | mutex_exit(&msipic_list_lock); |
274 | |
275 | return pic; |
276 | } |
277 | |
278 | static void |
279 | msipic_destruct_common_msi_pic(struct pic *msi_pic) |
280 | { |
281 | struct msipic *msipic; |
282 | |
283 | if (msi_pic == NULL) |
284 | return; |
285 | |
286 | msipic = msi_pic->pic_msipic; |
287 | mutex_enter(&msipic_list_lock); |
288 | LIST_REMOVE(msipic, mp_list); |
289 | msipic_release_common_msi_devid(msipic->mp_devid); |
290 | mutex_exit(&msipic_list_lock); |
291 | |
292 | kmem_free(msipic, sizeof(*msipic)); |
293 | kmem_free(msi_pic, sizeof(*msi_pic)); |
294 | } |
295 | |
296 | /* |
297 | * The pic is MSI/MSI-X pic or not. |
298 | */ |
299 | bool |
300 | msipic_is_msi_pic(struct pic *pic) |
301 | { |
302 | |
303 | return (pic->pic_msipic != NULL); |
304 | } |
305 | |
306 | /* |
307 | * Return the MSI/MSI-X devid which is unique for each devices. |
308 | */ |
309 | int |
310 | msipic_get_devid(struct pic *pic) |
311 | { |
312 | |
313 | KASSERT(msipic_is_msi_pic(pic)); |
314 | |
315 | return pic->pic_msipic->mp_devid; |
316 | } |
317 | |
318 | #define MSI_MSICTL_ENABLE 1 |
319 | #define MSI_MSICTL_DISABLE 0 |
320 | static void |
321 | msi_set_msictl_enablebit(struct pic *pic, int msi_vec, int flag) |
322 | { |
323 | pci_chipset_tag_t pc; |
324 | struct pci_attach_args *pa; |
325 | pcitag_t tag; |
326 | pcireg_t ctl; |
327 | int off, err __diagused; |
328 | |
329 | pc = NULL; |
330 | pa = &pic->pic_msipic->mp_pa; |
331 | tag = pa->pa_tag; |
332 | err = pci_get_capability(pc, tag, PCI_CAP_MSI, &off, NULL); |
333 | KASSERT(err != 0); |
334 | |
335 | /* |
336 | * MSI can establish only one vector at once. |
337 | * So, use whole device mask bit instead of a vector mask bit. |
338 | */ |
339 | ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL); |
340 | if (flag == MSI_MSICTL_ENABLE) |
341 | ctl |= PCI_MSI_CTL_MSI_ENABLE; |
342 | else |
343 | ctl &= ~PCI_MSI_CTL_MSI_ENABLE; |
344 | |
345 | pci_conf_write(pc, tag, off, ctl); |
346 | } |
347 | |
348 | static void |
349 | msi_hwmask(struct pic *pic, int msi_vec) |
350 | { |
351 | |
352 | msi_set_msictl_enablebit(pic, msi_vec, MSI_MSICTL_DISABLE); |
353 | } |
354 | |
355 | /* |
356 | * Do not use pic->hwunmask() immediately after pic->delroute(). |
357 | * It is required to use pic->addroute() before pic->hwunmask(). |
358 | */ |
359 | static void |
360 | msi_hwunmask(struct pic *pic, int msi_vec) |
361 | { |
362 | |
363 | msi_set_msictl_enablebit(pic, msi_vec, MSI_MSICTL_ENABLE); |
364 | } |
365 | |
366 | static void |
367 | msi_addroute(struct pic *pic, struct cpu_info *ci, |
368 | int unused, int idt_vec, int type) |
369 | { |
370 | pci_chipset_tag_t pc; |
371 | struct pci_attach_args *pa; |
372 | pcitag_t tag; |
373 | pcireg_t addr, data, ctl; |
374 | int off, err __diagused; |
375 | |
376 | pc = NULL; |
377 | pa = &pic->pic_msipic->mp_pa; |
378 | tag = pa->pa_tag; |
379 | err = pci_get_capability(pc, tag, PCI_CAP_MSI, &off, NULL); |
380 | KASSERT(err != 0); |
381 | |
382 | /* |
383 | * See Intel 64 and IA-32 Architectures Software Developer's Manual |
384 | * Volume 3 10.11 Message Signalled Interrupts. |
385 | */ |
386 | /* |
387 | * "cpuid" for MSI address is local APIC ID. In NetBSD, the ID is |
388 | * the same as ci->ci_cpuid. |
389 | */ |
390 | addr = LAPIC_MSIADDR_BASE | __SHIFTIN(ci->ci_cpuid, |
391 | LAPIC_MSIADDR_DSTID_MASK); |
392 | /* If trigger mode is edge, it don't care level for trigger mode. */ |
393 | data = __SHIFTIN(idt_vec, LAPIC_MSIDATA_VECTOR_MASK) |
394 | | LAPIC_MSIDATA_TRGMODE_EDGE | LAPIC_MSIDATA_DM_FIXED; |
395 | |
396 | ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL); |
397 | if (ctl & PCI_MSI_CTL_64BIT_ADDR) { |
398 | pci_conf_write(pc, tag, off + PCI_MSI_MADDR64_LO, addr); |
399 | pci_conf_write(pc, tag, off + PCI_MSI_MADDR64_HI, 0); |
400 | pci_conf_write(pc, tag, off + PCI_MSI_MDATA64, data); |
401 | } else { |
402 | pci_conf_write(pc, tag, off + PCI_MSI_MADDR, addr); |
403 | pci_conf_write(pc, tag, off + PCI_MSI_MDATA, data); |
404 | } |
405 | ctl |= PCI_MSI_CTL_MSI_ENABLE; |
406 | pci_conf_write(pc, tag, off + PCI_MSI_CTL, ctl); |
407 | } |
408 | |
409 | /* |
410 | * Do not use pic->hwunmask() immediately after pic->delroute(). |
411 | * It is required to use pic->addroute() before pic->hwunmask(). |
412 | */ |
413 | static void |
414 | msi_delroute(struct pic *pic, struct cpu_info *ci, |
415 | int msi_vec, int idt_vec, int type) |
416 | { |
417 | |
418 | msi_hwmask(pic, msi_vec); |
419 | } |
420 | |
421 | /* |
422 | * Template for MSI pic. |
423 | * .pic_msipic is set later in construct_msi_pic(). |
424 | */ |
425 | static struct pic msi_pic_tmpl = { |
426 | .pic_type = PIC_MSI, |
427 | .pic_vecbase = 0, |
428 | .pic_apicid = 0, |
429 | .pic_lock = __SIMPLELOCK_UNLOCKED, /* not used for msi_pic */ |
430 | .pic_hwmask = msi_hwmask, |
431 | .pic_hwunmask = msi_hwunmask, |
432 | .pic_addroute = msi_addroute, |
433 | .pic_delroute = msi_delroute, |
434 | .pic_edge_stubs = ioapic_edge_stubs, |
435 | .pic_ioapic = NULL, |
436 | }; |
437 | |
438 | /* |
439 | * Create pseudo pic for a MSI device. |
440 | */ |
441 | struct pic * |
442 | msipic_construct_msi_pic(const struct pci_attach_args *pa) |
443 | { |
444 | struct pic *msi_pic; |
445 | char pic_name_buf[MSIPICNAMEBUF]; |
446 | |
447 | msi_pic = msipic_construct_common_msi_pic(pa, &msi_pic_tmpl); |
448 | if (msi_pic == NULL) { |
449 | DPRINTF(("cannot allocate MSI pic.\n" )); |
450 | return NULL; |
451 | } |
452 | |
453 | memset(pic_name_buf, 0, MSIPICNAMEBUF); |
454 | snprintf(pic_name_buf, MSIPICNAMEBUF, "msi%d" , |
455 | msi_pic->pic_msipic->mp_devid); |
456 | strncpy(msi_pic->pic_msipic->mp_pic_name, pic_name_buf, |
457 | MSIPICNAMEBUF - 1); |
458 | msi_pic->pic_name = msi_pic->pic_msipic->mp_pic_name; |
459 | |
460 | return msi_pic; |
461 | } |
462 | |
463 | /* |
464 | * Delete pseudo pic for a MSI device. |
465 | */ |
466 | void |
467 | msipic_destruct_msi_pic(struct pic *msi_pic) |
468 | { |
469 | |
470 | msipic_destruct_common_msi_pic(msi_pic); |
471 | } |
472 | |
473 | #define MSIX_VECCTL_HWMASK 1 |
474 | #define MSIX_VECCTL_HWUNMASK 0 |
475 | static void |
476 | msix_set_vecctl_mask(struct pic *pic, int msix_vec, int flag) |
477 | { |
478 | bus_space_tag_t bstag; |
479 | bus_space_handle_t bshandle; |
480 | uint64_t entry_base; |
481 | uint32_t vecctl; |
482 | |
483 | if (msix_vec < 0) { |
484 | DPRINTF(("%s: invalid MSI-X table index, devid=%d vecid=%d" , |
485 | __func__, msipic_get_devid(pic), msix_vec)); |
486 | return; |
487 | } |
488 | |
489 | entry_base = PCI_MSIX_TABLE_ENTRY_SIZE * msix_vec; |
490 | |
491 | bstag = pic->pic_msipic->mp_bstag; |
492 | bshandle = pic->pic_msipic->mp_bshandle; |
493 | vecctl = bus_space_read_4(bstag, bshandle, |
494 | entry_base + PCI_MSIX_TABLE_ENTRY_VECTCTL); |
495 | if (flag == MSIX_VECCTL_HWMASK) |
496 | vecctl |= PCI_MSIX_VECTCTL_MASK; |
497 | else |
498 | vecctl &= ~PCI_MSIX_VECTCTL_MASK; |
499 | |
500 | bus_space_write_4(bstag, bshandle, |
501 | entry_base + PCI_MSIX_TABLE_ENTRY_VECTCTL, vecctl); |
502 | BUS_SPACE_WRITE_FLUSH(bstag, bshandle); |
503 | } |
504 | |
505 | static void |
506 | msix_hwmask(struct pic *pic, int msix_vec) |
507 | { |
508 | |
509 | msix_set_vecctl_mask(pic, msix_vec, MSIX_VECCTL_HWMASK); |
510 | } |
511 | |
512 | /* |
513 | * Do not use pic->hwunmask() immediately after pic->delroute(). |
514 | * It is required to use pic->addroute() before pic->hwunmask(). |
515 | */ |
516 | static void |
517 | msix_hwunmask(struct pic *pic, int msix_vec) |
518 | { |
519 | |
520 | msix_set_vecctl_mask(pic, msix_vec, MSIX_VECCTL_HWUNMASK); |
521 | } |
522 | |
523 | static void |
524 | msix_addroute(struct pic *pic, struct cpu_info *ci, |
525 | int msix_vec, int idt_vec, int type) |
526 | { |
527 | pci_chipset_tag_t pc; |
528 | struct pci_attach_args *pa; |
529 | pcitag_t tag; |
530 | bus_space_tag_t bstag; |
531 | bus_space_handle_t bshandle; |
532 | uint64_t entry_base; |
533 | pcireg_t addr, data, ctl; |
534 | int off, err __diagused; |
535 | |
536 | if (msix_vec < 0) { |
537 | DPRINTF(("%s: invalid MSI-X table index, devid=%d vecid=%d" , |
538 | __func__, msipic_get_devid(pic), msix_vec)); |
539 | return; |
540 | } |
541 | |
542 | pa = &pic->pic_msipic->mp_pa; |
543 | pc = pa->pa_pc; |
544 | tag = pa->pa_tag; |
545 | err = pci_get_capability(pc, tag, PCI_CAP_MSIX, &off, NULL); |
546 | KASSERT(err != 0); |
547 | |
548 | entry_base = PCI_MSIX_TABLE_ENTRY_SIZE * msix_vec; |
549 | |
550 | /* |
551 | * See Intel 64 and IA-32 Architectures Software Developer's Manual |
552 | * Volume 3 10.11 Message Signalled Interrupts. |
553 | */ |
554 | /* |
555 | * "cpuid" for MSI-X address is local APIC ID. In NetBSD, the ID is |
556 | * the same as ci->ci_cpuid. |
557 | */ |
558 | addr = LAPIC_MSIADDR_BASE | __SHIFTIN(ci->ci_cpuid, |
559 | LAPIC_MSIADDR_DSTID_MASK); |
560 | /* If trigger mode is edge, it don't care level for trigger mode. */ |
561 | data = __SHIFTIN(idt_vec, LAPIC_MSIDATA_VECTOR_MASK) |
562 | | LAPIC_MSIDATA_TRGMODE_EDGE | LAPIC_MSIDATA_DM_FIXED; |
563 | |
564 | bstag = pic->pic_msipic->mp_bstag; |
565 | bshandle = pic->pic_msipic->mp_bshandle; |
566 | bus_space_write_4(bstag, bshandle, |
567 | entry_base + PCI_MSIX_TABLE_ENTRY_ADDR_LO, addr); |
568 | bus_space_write_4(bstag, bshandle, |
569 | entry_base + PCI_MSIX_TABLE_ENTRY_ADDR_HI, 0); |
570 | bus_space_write_4(bstag, bshandle, |
571 | entry_base + PCI_MSIX_TABLE_ENTRY_DATA, data); |
572 | bus_space_write_4(bstag, bshandle, |
573 | entry_base + PCI_MSIX_TABLE_ENTRY_VECTCTL, 0); |
574 | BUS_SPACE_WRITE_FLUSH(bstag, bshandle); |
575 | |
576 | ctl = pci_conf_read(pc, tag, off + PCI_MSIX_CTL); |
577 | ctl |= PCI_MSIX_CTL_ENABLE; |
578 | pci_conf_write(pc, tag, off + PCI_MSIX_CTL, ctl); |
579 | } |
580 | |
581 | /* |
582 | * Do not use pic->hwunmask() immediately after pic->delroute(). |
583 | * It is required to use pic->addroute() before pic->hwunmask(). |
584 | */ |
585 | static void |
586 | msix_delroute(struct pic *pic, struct cpu_info *ci, |
587 | int msix_vec, int vec, int type) |
588 | { |
589 | |
590 | msix_hwmask(pic, msix_vec); |
591 | } |
592 | |
593 | /* |
594 | * Template for MSI-X pic. |
595 | * .pic_msipic is set later in construct_msix_pic(). |
596 | */ |
597 | static struct pic msix_pic_tmpl = { |
598 | .pic_type = PIC_MSIX, |
599 | .pic_vecbase = 0, |
600 | .pic_apicid = 0, |
601 | .pic_lock = __SIMPLELOCK_UNLOCKED, /* not used for msix_pic */ |
602 | .pic_hwmask = msix_hwmask, |
603 | .pic_hwunmask = msix_hwunmask, |
604 | .pic_addroute = msix_addroute, |
605 | .pic_delroute = msix_delroute, |
606 | .pic_edge_stubs = ioapic_edge_stubs, |
607 | }; |
608 | |
609 | struct pic * |
610 | msipic_construct_msix_pic(const struct pci_attach_args *pa) |
611 | { |
612 | struct pic *msix_pic; |
613 | pci_chipset_tag_t pc; |
614 | pcitag_t tag; |
615 | pcireg_t tbl; |
616 | bus_space_tag_t bstag; |
617 | bus_space_handle_t bshandle; |
618 | bus_size_t bssize; |
619 | size_t table_size; |
620 | uint32_t table_offset; |
621 | u_int memtype; |
622 | bus_addr_t memaddr; |
623 | int flags; |
624 | int bir, bar, err, off, table_nentry; |
625 | char pic_name_buf[MSIPICNAMEBUF]; |
626 | |
627 | table_nentry = pci_msix_count(pa->pa_pc, pa->pa_tag); |
628 | if (table_nentry == 0) { |
629 | DPRINTF(("MSI-X table entry is 0.\n" )); |
630 | return NULL; |
631 | } |
632 | |
633 | pc = pa->pa_pc; |
634 | tag = pa->pa_tag; |
635 | if (pci_get_capability(pc, tag, PCI_CAP_MSIX, &off, NULL) == 0) { |
636 | DPRINTF(("%s: no msix capability" , __func__)); |
637 | return NULL; |
638 | } |
639 | |
640 | msix_pic = msipic_construct_common_msi_pic(pa, &msix_pic_tmpl); |
641 | if (msix_pic == NULL) { |
642 | DPRINTF(("cannot allocate MSI-X pic.\n" )); |
643 | return NULL; |
644 | } |
645 | |
646 | memset(pic_name_buf, 0, MSIPICNAMEBUF); |
647 | snprintf(pic_name_buf, MSIPICNAMEBUF, "msix%d" , |
648 | msix_pic->pic_msipic->mp_devid); |
649 | strncpy(msix_pic->pic_msipic->mp_pic_name, pic_name_buf, |
650 | MSIPICNAMEBUF - 1); |
651 | msix_pic->pic_name = msix_pic->pic_msipic->mp_pic_name; |
652 | |
653 | tbl = pci_conf_read(pc, tag, off + PCI_MSIX_TBLOFFSET); |
654 | table_offset = tbl & PCI_MSIX_TBLOFFSET_MASK; |
655 | bir = tbl & PCI_MSIX_PBABIR_MASK; |
656 | switch(bir) { |
657 | case 0: |
658 | bar = PCI_BAR0; |
659 | break; |
660 | case 1: |
661 | bar = PCI_BAR1; |
662 | break; |
663 | case 2: |
664 | bar = PCI_BAR2; |
665 | break; |
666 | case 3: |
667 | bar = PCI_BAR3; |
668 | break; |
669 | case 4: |
670 | bar = PCI_BAR4; |
671 | break; |
672 | case 5: |
673 | bar = PCI_BAR5; |
674 | break; |
675 | default: |
676 | aprint_error("detect an illegal device! The device use reserved BIR values.\n" ); |
677 | msipic_destruct_common_msi_pic(msix_pic); |
678 | return NULL; |
679 | } |
680 | memtype = pci_mapreg_type(pc, tag, bar); |
681 | /* |
682 | * PCI_MSIX_TABLE_ENTRY_SIZE consists below |
683 | * - Vector Control (32bit) |
684 | * - Message Data (32bit) |
685 | * - Message Upper Address (32bit) |
686 | * - Message Lower Address (32bit) |
687 | */ |
688 | table_size = table_nentry * PCI_MSIX_TABLE_ENTRY_SIZE; |
689 | #if 0 |
690 | err = pci_mapreg_submap(pa, bar, memtype, BUS_SPACE_MAP_LINEAR, |
691 | roundup(table_size, PAGE_SIZE), table_offset, |
692 | &bstag, &bshandle, NULL, &bssize); |
693 | #else |
694 | /* |
695 | * Workaround for PCI prefetchable bit. Some chips (e.g. Intel 82599) |
696 | * report SERR and MSI-X doesn't work. This problem might not be the |
697 | * driver's bug but our PCI common part or VMs' bug. Until we find a |
698 | * real reason, we ignore the prefetchable bit. |
699 | */ |
700 | if (pci_mapreg_info(pa->pa_pc, pa->pa_tag, bar, memtype, |
701 | &memaddr, NULL, &flags) != 0) { |
702 | DPRINTF(("cannot get a map info.\n" )); |
703 | msipic_destruct_common_msi_pic(msix_pic); |
704 | return NULL; |
705 | } |
706 | if ((flags & BUS_SPACE_MAP_PREFETCHABLE) != 0) { |
707 | DPRINTF(( "clear prefetchable bit\n" )); |
708 | flags &= ~BUS_SPACE_MAP_PREFETCHABLE; |
709 | } |
710 | bssize = roundup(table_size, PAGE_SIZE); |
711 | err = bus_space_map(pa->pa_memt, memaddr + table_offset, bssize, flags, |
712 | &bshandle); |
713 | bstag = pa->pa_memt; |
714 | #endif |
715 | if (err) { |
716 | DPRINTF(("cannot map msix table.\n" )); |
717 | msipic_destruct_common_msi_pic(msix_pic); |
718 | return NULL; |
719 | } |
720 | msix_pic->pic_msipic->mp_bstag = bstag; |
721 | msix_pic->pic_msipic->mp_bshandle = bshandle; |
722 | msix_pic->pic_msipic->mp_bssize = bssize; |
723 | |
724 | return msix_pic; |
725 | } |
726 | |
727 | /* |
728 | * Delete pseudo pic for a MSI-X device. |
729 | */ |
730 | void |
731 | msipic_destruct_msix_pic(struct pic *msix_pic) |
732 | { |
733 | struct msipic *msipic; |
734 | |
735 | KASSERT(msipic_is_msi_pic(msix_pic)); |
736 | KASSERT(msix_pic->pic_type == PIC_MSIX); |
737 | |
738 | msipic = msix_pic->pic_msipic; |
739 | bus_space_unmap(msipic->mp_bstag, msipic->mp_bshandle, |
740 | msipic->mp_bssize); |
741 | |
742 | msipic_destruct_common_msi_pic(msix_pic); |
743 | } |
744 | |
745 | /* |
746 | * Set the number of MSI vectors for pseudo MSI pic. |
747 | */ |
748 | int |
749 | msipic_set_msi_vectors(struct pic *msi_pic, pci_intr_handle_t *pihs, |
750 | int count) |
751 | { |
752 | |
753 | KASSERT(msipic_is_msi_pic(msi_pic)); |
754 | |
755 | msi_pic->pic_msipic->mp_veccnt = count; |
756 | return 0; |
757 | } |
758 | |
759 | /* |
760 | * Initialize the system to use MSI/MSI-X. |
761 | */ |
762 | void |
763 | msipic_init(void) |
764 | { |
765 | |
766 | mutex_init(&msipic_list_lock, MUTEX_DEFAULT, IPL_NONE); |
767 | } |
768 | |