1/* $NetBSD: patch.c,v 1.22 2013/11/15 08:47:55 msaitoh Exp $ */
2
3/*-
4 * Copyright (c) 2007, 2008, 2009 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Andrew Doran.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32/*
33 * Patch kernel code at boot time, depending on available CPU features.
34 */
35
36#include <sys/cdefs.h>
37__KERNEL_RCSID(0, "$NetBSD: patch.c,v 1.22 2013/11/15 08:47:55 msaitoh Exp $");
38
39#include "opt_lockdebug.h"
40#ifdef i386
41#include "opt_spldebug.h"
42#endif
43
44#include <sys/types.h>
45#include <sys/systm.h>
46
47#include <machine/cpu.h>
48#include <machine/cpufunc.h>
49#include <machine/specialreg.h>
50
51#include <x86/cpuvar.h>
52#include <x86/cputypes.h>
53
54void spllower(int);
55void spllower_end(void);
56void cx8_spllower(int);
57void cx8_spllower_end(void);
58void cx8_spllower_patch(void);
59
60void mutex_spin_exit_end(void);
61void i686_mutex_spin_exit(int);
62void i686_mutex_spin_exit_end(void);
63void i686_mutex_spin_exit_patch(void);
64
65void membar_consumer(void);
66void membar_consumer_end(void);
67void membar_sync(void);
68void membar_sync_end(void);
69void sse2_lfence(void);
70void sse2_lfence_end(void);
71void sse2_mfence(void);
72void sse2_mfence_end(void);
73
74void _atomic_cas_64(void);
75void _atomic_cas_64_end(void);
76void _atomic_cas_cx8(void);
77void _atomic_cas_cx8_end(void);
78
79extern void *x86_lockpatch[];
80extern void *x86_retpatch[];
81extern void *atomic_lockpatch[];
82
83#define X86_NOP 0x90
84#define X86_REP 0xf3
85#define X86_RET 0xc3
86#define X86_CS 0x2e
87#define X86_DS 0x3e
88#define X86_GROUP_0F 0x0f
89
90static void
91adjust_jumpoff(uint8_t *ptr, void *from_s, void *to_s)
92{
93
94 /* Branch hints */
95 if (ptr[0] == X86_CS || ptr[0] == X86_DS)
96 ptr++;
97 /* Conditional jumps */
98 if (ptr[0] == X86_GROUP_0F)
99 ptr++;
100 /* 4-byte relative jump or call */
101 *(uint32_t *)(ptr + 1 - (uintptr_t)from_s + (uintptr_t)to_s) +=
102 ((uint32_t)(uintptr_t)from_s - (uint32_t)(uintptr_t)to_s);
103}
104
105static void __unused
106patchfunc(void *from_s, void *from_e, void *to_s, void *to_e,
107 void *pcrel)
108{
109
110 if ((uintptr_t)from_e - (uintptr_t)from_s !=
111 (uintptr_t)to_e - (uintptr_t)to_s)
112 panic("patchfunc: sizes do not match (from=%p)", from_s);
113
114 memcpy(to_s, from_s, (uintptr_t)to_e - (uintptr_t)to_s);
115 if (pcrel != NULL)
116 adjust_jumpoff(pcrel, from_s, to_s);
117
118#ifdef GPROF
119#ifdef i386
120#define MCOUNT_CALL_OFFSET 3
121#endif
122#ifdef __x86_64__
123#define MCOUNT_CALL_OFFSET 5
124#endif
125 /* Patch mcount call offset */
126 adjust_jumpoff((uint8_t *)from_s + MCOUNT_CALL_OFFSET, from_s, to_s);
127#endif
128}
129
130static inline void __unused
131patchbytes(void *addr, const int byte1, const int byte2, const int byte3)
132{
133
134 ((uint8_t *)addr)[0] = (uint8_t)byte1;
135 if (byte2 != -1)
136 ((uint8_t *)addr)[1] = (uint8_t)byte2;
137 if (byte3 != -1)
138 ((uint8_t *)addr)[2] = (uint8_t)byte3;
139}
140
141void
142x86_patch(bool early)
143{
144 static bool first, second;
145 u_long psl;
146 u_long cr0;
147 int i;
148
149 if (early) {
150 if (first)
151 return;
152 first = true;
153 } else {
154 if (second)
155 return;
156 second = true;
157 }
158
159 /* Disable interrupts. */
160 psl = x86_read_psl();
161 x86_disable_intr();
162
163 /* Disable write protection in supervisor mode. */
164 cr0 = rcr0();
165 lcr0(cr0 & ~CR0_WP);
166
167#if !defined(GPROF)
168 if (!early && ncpu == 1) {
169#ifndef LOCKDEBUG
170 /* Uniprocessor: kill LOCK prefixes. */
171 for (i = 0; x86_lockpatch[i] != 0; i++)
172 patchbytes(x86_lockpatch[i], X86_NOP, -1, -1);
173 for (i = 0; atomic_lockpatch[i] != 0; i++)
174 patchbytes(atomic_lockpatch[i], X86_NOP, -1, -1);
175#endif /* !LOCKDEBUG */
176 }
177 if (!early && (cpu_feature[0] & CPUID_SSE2) != 0) {
178 /* Faster memory barriers. */
179 patchfunc(
180 sse2_lfence, sse2_lfence_end,
181 membar_consumer, membar_consumer_end,
182 NULL
183 );
184 patchfunc(
185 sse2_mfence, sse2_mfence_end,
186 membar_sync, membar_sync_end,
187 NULL
188 );
189 }
190#endif /* GPROF */
191
192#ifdef i386
193 /*
194 * Patch early and late. Second time around the 'lock' prefix
195 * may be gone.
196 */
197 if ((cpu_feature[0] & CPUID_CX8) != 0) {
198 patchfunc(
199 _atomic_cas_cx8, _atomic_cas_cx8_end,
200 _atomic_cas_64, _atomic_cas_64_end,
201 NULL
202 );
203 }
204#endif /* i386 */
205
206#if !defined(SPLDEBUG)
207 if (!early && (cpu_feature[0] & CPUID_CX8) != 0) {
208 /* Faster splx(), mutex_spin_exit(). */
209 patchfunc(
210 cx8_spllower, cx8_spllower_end,
211 spllower, spllower_end,
212 cx8_spllower_patch
213 );
214#if defined(i386) && !defined(LOCKDEBUG)
215 patchfunc(
216 i686_mutex_spin_exit, i686_mutex_spin_exit_end,
217 mutex_spin_exit, mutex_spin_exit_end,
218 i686_mutex_spin_exit_patch
219 );
220#endif /* i386 && !LOCKDEBUG */
221 }
222#endif /* !SPLDEBUG */
223
224 /*
225 * On some Opteron revisions, locked operations erroneously
226 * allow memory references to be `bled' outside of critical
227 * sections. Apply workaround.
228 */
229 if (cpu_vendor == CPUVENDOR_AMD &&
230 (CPUID_TO_FAMILY(cpu_info_primary.ci_signature) == 0xe ||
231 (CPUID_TO_FAMILY(cpu_info_primary.ci_signature) == 0xf &&
232 CPUID_TO_EXTMODEL(cpu_info_primary.ci_signature) < 0x4))) {
233 for (i = 0; x86_retpatch[i] != 0; i++) {
234 /* ret,nop,nop,ret -> lfence,ret */
235 patchbytes(x86_retpatch[i], 0x0f, 0xae, 0xe8);
236 }
237 }
238
239 /* Write back and invalidate cache, flush pipelines. */
240 wbinvd();
241 x86_flush();
242 x86_write_psl(psl);
243
244 /* Re-enable write protection. */
245 lcr0(cr0);
246}
247