1 | /* $NetBSD: patch.c,v 1.22 2013/11/15 08:47:55 msaitoh Exp $ */ |
2 | |
3 | /*- |
4 | * Copyright (c) 2007, 2008, 2009 The NetBSD Foundation, Inc. |
5 | * All rights reserved. |
6 | * |
7 | * This code is derived from software contributed to The NetBSD Foundation |
8 | * by Andrew Doran. |
9 | * |
10 | * Redistribution and use in source and binary forms, with or without |
11 | * modification, are permitted provided that the following conditions |
12 | * are met: |
13 | * 1. Redistributions of source code must retain the above copyright |
14 | * notice, this list of conditions and the following disclaimer. |
15 | * 2. Redistributions in binary form must reproduce the above copyright |
16 | * notice, this list of conditions and the following disclaimer in the |
17 | * documentation and/or other materials provided with the distribution. |
18 | * |
19 | * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS |
20 | * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED |
21 | * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
22 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS |
23 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
24 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
25 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS |
26 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN |
27 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
28 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
29 | * POSSIBILITY OF SUCH DAMAGE. |
30 | */ |
31 | |
32 | /* |
33 | * Patch kernel code at boot time, depending on available CPU features. |
34 | */ |
35 | |
36 | #include <sys/cdefs.h> |
37 | __KERNEL_RCSID(0, "$NetBSD: patch.c,v 1.22 2013/11/15 08:47:55 msaitoh Exp $" ); |
38 | |
39 | #include "opt_lockdebug.h" |
40 | #ifdef i386 |
41 | #include "opt_spldebug.h" |
42 | #endif |
43 | |
44 | #include <sys/types.h> |
45 | #include <sys/systm.h> |
46 | |
47 | #include <machine/cpu.h> |
48 | #include <machine/cpufunc.h> |
49 | #include <machine/specialreg.h> |
50 | |
51 | #include <x86/cpuvar.h> |
52 | #include <x86/cputypes.h> |
53 | |
54 | void spllower(int); |
55 | void spllower_end(void); |
56 | void cx8_spllower(int); |
57 | void cx8_spllower_end(void); |
58 | void cx8_spllower_patch(void); |
59 | |
60 | void mutex_spin_exit_end(void); |
61 | void i686_mutex_spin_exit(int); |
62 | void i686_mutex_spin_exit_end(void); |
63 | void i686_mutex_spin_exit_patch(void); |
64 | |
65 | void membar_consumer(void); |
66 | void membar_consumer_end(void); |
67 | void membar_sync(void); |
68 | void membar_sync_end(void); |
69 | void sse2_lfence(void); |
70 | void sse2_lfence_end(void); |
71 | void sse2_mfence(void); |
72 | void sse2_mfence_end(void); |
73 | |
74 | void _atomic_cas_64(void); |
75 | void _atomic_cas_64_end(void); |
76 | void _atomic_cas_cx8(void); |
77 | void _atomic_cas_cx8_end(void); |
78 | |
79 | extern void *x86_lockpatch[]; |
80 | extern void *x86_retpatch[]; |
81 | extern void *atomic_lockpatch[]; |
82 | |
83 | #define X86_NOP 0x90 |
84 | #define X86_REP 0xf3 |
85 | #define X86_RET 0xc3 |
86 | #define X86_CS 0x2e |
87 | #define X86_DS 0x3e |
88 | #define X86_GROUP_0F 0x0f |
89 | |
90 | static void |
91 | adjust_jumpoff(uint8_t *ptr, void *from_s, void *to_s) |
92 | { |
93 | |
94 | /* Branch hints */ |
95 | if (ptr[0] == X86_CS || ptr[0] == X86_DS) |
96 | ptr++; |
97 | /* Conditional jumps */ |
98 | if (ptr[0] == X86_GROUP_0F) |
99 | ptr++; |
100 | /* 4-byte relative jump or call */ |
101 | *(uint32_t *)(ptr + 1 - (uintptr_t)from_s + (uintptr_t)to_s) += |
102 | ((uint32_t)(uintptr_t)from_s - (uint32_t)(uintptr_t)to_s); |
103 | } |
104 | |
105 | static void __unused |
106 | patchfunc(void *from_s, void *from_e, void *to_s, void *to_e, |
107 | void *pcrel) |
108 | { |
109 | |
110 | if ((uintptr_t)from_e - (uintptr_t)from_s != |
111 | (uintptr_t)to_e - (uintptr_t)to_s) |
112 | panic("patchfunc: sizes do not match (from=%p)" , from_s); |
113 | |
114 | memcpy(to_s, from_s, (uintptr_t)to_e - (uintptr_t)to_s); |
115 | if (pcrel != NULL) |
116 | adjust_jumpoff(pcrel, from_s, to_s); |
117 | |
118 | #ifdef GPROF |
119 | #ifdef i386 |
120 | #define MCOUNT_CALL_OFFSET 3 |
121 | #endif |
122 | #ifdef __x86_64__ |
123 | #define MCOUNT_CALL_OFFSET 5 |
124 | #endif |
125 | /* Patch mcount call offset */ |
126 | adjust_jumpoff((uint8_t *)from_s + MCOUNT_CALL_OFFSET, from_s, to_s); |
127 | #endif |
128 | } |
129 | |
130 | static inline void __unused |
131 | patchbytes(void *addr, const int byte1, const int byte2, const int byte3) |
132 | { |
133 | |
134 | ((uint8_t *)addr)[0] = (uint8_t)byte1; |
135 | if (byte2 != -1) |
136 | ((uint8_t *)addr)[1] = (uint8_t)byte2; |
137 | if (byte3 != -1) |
138 | ((uint8_t *)addr)[2] = (uint8_t)byte3; |
139 | } |
140 | |
141 | void |
142 | x86_patch(bool early) |
143 | { |
144 | static bool first, second; |
145 | u_long psl; |
146 | u_long cr0; |
147 | int i; |
148 | |
149 | if (early) { |
150 | if (first) |
151 | return; |
152 | first = true; |
153 | } else { |
154 | if (second) |
155 | return; |
156 | second = true; |
157 | } |
158 | |
159 | /* Disable interrupts. */ |
160 | psl = x86_read_psl(); |
161 | x86_disable_intr(); |
162 | |
163 | /* Disable write protection in supervisor mode. */ |
164 | cr0 = rcr0(); |
165 | lcr0(cr0 & ~CR0_WP); |
166 | |
167 | #if !defined(GPROF) |
168 | if (!early && ncpu == 1) { |
169 | #ifndef LOCKDEBUG |
170 | /* Uniprocessor: kill LOCK prefixes. */ |
171 | for (i = 0; x86_lockpatch[i] != 0; i++) |
172 | patchbytes(x86_lockpatch[i], X86_NOP, -1, -1); |
173 | for (i = 0; atomic_lockpatch[i] != 0; i++) |
174 | patchbytes(atomic_lockpatch[i], X86_NOP, -1, -1); |
175 | #endif /* !LOCKDEBUG */ |
176 | } |
177 | if (!early && (cpu_feature[0] & CPUID_SSE2) != 0) { |
178 | /* Faster memory barriers. */ |
179 | patchfunc( |
180 | sse2_lfence, sse2_lfence_end, |
181 | membar_consumer, membar_consumer_end, |
182 | NULL |
183 | ); |
184 | patchfunc( |
185 | sse2_mfence, sse2_mfence_end, |
186 | membar_sync, membar_sync_end, |
187 | NULL |
188 | ); |
189 | } |
190 | #endif /* GPROF */ |
191 | |
192 | #ifdef i386 |
193 | /* |
194 | * Patch early and late. Second time around the 'lock' prefix |
195 | * may be gone. |
196 | */ |
197 | if ((cpu_feature[0] & CPUID_CX8) != 0) { |
198 | patchfunc( |
199 | _atomic_cas_cx8, _atomic_cas_cx8_end, |
200 | _atomic_cas_64, _atomic_cas_64_end, |
201 | NULL |
202 | ); |
203 | } |
204 | #endif /* i386 */ |
205 | |
206 | #if !defined(SPLDEBUG) |
207 | if (!early && (cpu_feature[0] & CPUID_CX8) != 0) { |
208 | /* Faster splx(), mutex_spin_exit(). */ |
209 | patchfunc( |
210 | cx8_spllower, cx8_spllower_end, |
211 | spllower, spllower_end, |
212 | cx8_spllower_patch |
213 | ); |
214 | #if defined(i386) && !defined(LOCKDEBUG) |
215 | patchfunc( |
216 | i686_mutex_spin_exit, i686_mutex_spin_exit_end, |
217 | mutex_spin_exit, mutex_spin_exit_end, |
218 | i686_mutex_spin_exit_patch |
219 | ); |
220 | #endif /* i386 && !LOCKDEBUG */ |
221 | } |
222 | #endif /* !SPLDEBUG */ |
223 | |
224 | /* |
225 | * On some Opteron revisions, locked operations erroneously |
226 | * allow memory references to be `bled' outside of critical |
227 | * sections. Apply workaround. |
228 | */ |
229 | if (cpu_vendor == CPUVENDOR_AMD && |
230 | (CPUID_TO_FAMILY(cpu_info_primary.ci_signature) == 0xe || |
231 | (CPUID_TO_FAMILY(cpu_info_primary.ci_signature) == 0xf && |
232 | CPUID_TO_EXTMODEL(cpu_info_primary.ci_signature) < 0x4))) { |
233 | for (i = 0; x86_retpatch[i] != 0; i++) { |
234 | /* ret,nop,nop,ret -> lfence,ret */ |
235 | patchbytes(x86_retpatch[i], 0x0f, 0xae, 0xe8); |
236 | } |
237 | } |
238 | |
239 | /* Write back and invalidate cache, flush pipelines. */ |
240 | wbinvd(); |
241 | x86_flush(); |
242 | x86_write_psl(psl); |
243 | |
244 | /* Re-enable write protection. */ |
245 | lcr0(cr0); |
246 | } |
247 | |