1 | /* $NetBSD: lock.h,v 1.27 2013/01/22 22:09:44 christos Exp $ */ |
2 | |
3 | /*- |
4 | * Copyright (c) 2000, 2006 The NetBSD Foundation, Inc. |
5 | * All rights reserved. |
6 | * |
7 | * This code is derived from software contributed to The NetBSD Foundation |
8 | * by Jason R. Thorpe and Andrew Doran. |
9 | * |
10 | * Redistribution and use in source and binary forms, with or without |
11 | * modification, are permitted provided that the following conditions |
12 | * are met: |
13 | * 1. Redistributions of source code must retain the above copyright |
14 | * notice, this list of conditions and the following disclaimer. |
15 | * 2. Redistributions in binary form must reproduce the above copyright |
16 | * notice, this list of conditions and the following disclaimer in the |
17 | * documentation and/or other materials provided with the distribution. |
18 | * |
19 | * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS |
20 | * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED |
21 | * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
22 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS |
23 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
24 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
25 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS |
26 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN |
27 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
28 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
29 | * POSSIBILITY OF SUCH DAMAGE. |
30 | */ |
31 | |
32 | /* |
33 | * Machine-dependent spin lock operations. |
34 | */ |
35 | |
36 | #ifndef _X86_LOCK_H_ |
37 | #define _X86_LOCK_H_ |
38 | |
39 | #include <sys/param.h> |
40 | |
41 | static __inline int |
42 | __SIMPLELOCK_LOCKED_P(__cpu_simple_lock_t *__ptr) |
43 | { |
44 | return *__ptr == __SIMPLELOCK_LOCKED; |
45 | } |
46 | |
47 | static __inline int |
48 | __SIMPLELOCK_UNLOCKED_P(__cpu_simple_lock_t *__ptr) |
49 | { |
50 | return *__ptr == __SIMPLELOCK_UNLOCKED; |
51 | } |
52 | |
53 | static __inline void |
54 | __cpu_simple_lock_set(__cpu_simple_lock_t *__ptr) |
55 | { |
56 | |
57 | *__ptr = __SIMPLELOCK_LOCKED; |
58 | } |
59 | |
60 | static __inline void |
61 | __cpu_simple_lock_clear(__cpu_simple_lock_t *__ptr) |
62 | { |
63 | |
64 | *__ptr = __SIMPLELOCK_UNLOCKED; |
65 | } |
66 | |
67 | #ifdef _HARDKERNEL |
68 | # include <machine/cpufunc.h> |
69 | # define SPINLOCK_SPIN_HOOK /* nothing */ |
70 | # ifdef SPINLOCK_BACKOFF_HOOK |
71 | # undef SPINLOCK_BACKOFF_HOOK |
72 | # endif |
73 | # define SPINLOCK_BACKOFF_HOOK x86_pause() |
74 | # define SPINLOCK_INLINE |
75 | #else /* !_HARDKERNEL */ |
76 | # define SPINLOCK_BODY |
77 | # define SPINLOCK_INLINE static __inline __unused |
78 | #endif /* _HARDKERNEL */ |
79 | |
80 | SPINLOCK_INLINE void __cpu_simple_lock_init(__cpu_simple_lock_t *); |
81 | SPINLOCK_INLINE void __cpu_simple_lock(__cpu_simple_lock_t *); |
82 | SPINLOCK_INLINE int __cpu_simple_lock_try(__cpu_simple_lock_t *); |
83 | SPINLOCK_INLINE void __cpu_simple_unlock(__cpu_simple_lock_t *); |
84 | |
85 | #ifdef SPINLOCK_BODY |
86 | SPINLOCK_INLINE void |
87 | __cpu_simple_lock_init(__cpu_simple_lock_t *lockp) |
88 | { |
89 | |
90 | *lockp = __SIMPLELOCK_UNLOCKED; |
91 | __insn_barrier(); |
92 | } |
93 | |
94 | SPINLOCK_INLINE int |
95 | __cpu_simple_lock_try(__cpu_simple_lock_t *lockp) |
96 | { |
97 | uint8_t val; |
98 | |
99 | val = __SIMPLELOCK_LOCKED; |
100 | __asm volatile ("xchgb %0,(%2)" : |
101 | "=qQ" (val) |
102 | :"0" (val), "r" (lockp)); |
103 | __insn_barrier(); |
104 | return val == __SIMPLELOCK_UNLOCKED; |
105 | } |
106 | |
107 | SPINLOCK_INLINE void |
108 | __cpu_simple_lock(__cpu_simple_lock_t *lockp) |
109 | { |
110 | |
111 | while (!__cpu_simple_lock_try(lockp)) |
112 | /* nothing */; |
113 | __insn_barrier(); |
114 | } |
115 | |
116 | /* |
117 | * Note on x86 memory ordering |
118 | * |
119 | * When releasing a lock we must ensure that no stores or loads from within |
120 | * the critical section are re-ordered by the CPU to occur outside of it: |
121 | * they must have completed and be visible to other processors once the lock |
122 | * has been released. |
123 | * |
124 | * NetBSD usually runs with the kernel mapped (via MTRR) in a WB (write |
125 | * back) memory region. In that case, memory ordering on x86 platforms |
126 | * looks like this: |
127 | * |
128 | * i386 All loads/stores occur in instruction sequence. |
129 | * |
130 | * i486 All loads/stores occur in instruction sequence. In |
131 | * Pentium exceptional circumstances, loads can be re-ordered around |
132 | * stores, but for the purposes of releasing a lock it does |
133 | * not matter. Stores may not be immediately visible to other |
134 | * processors as they can be buffered. However, since the |
135 | * stores are buffered in order the lock release will always be |
136 | * the last operation in the critical section that becomes |
137 | * visible to other CPUs. |
138 | * |
139 | * Pentium Pro The "Intel 64 and IA-32 Architectures Software Developer's |
140 | * onwards Manual" volume 3A (order number 248966) says that (1) "Reads |
141 | * can be carried out speculatively and in any order" and (2) |
142 | * "Reads can pass buffered stores, but the processor is |
143 | * self-consistent.". This would be a problem for the below, |
144 | * and would mandate a locked instruction cycle or load fence |
145 | * before releasing the simple lock. |
146 | * |
147 | * The "Intel Pentium 4 Processor Optimization" guide (order |
148 | * number 253668-022US) says: "Loads can be moved before stores |
149 | * that occurred earlier in the program if they are not |
150 | * predicted to load from the same linear address.". This is |
151 | * not a problem since the only loads that can be re-ordered |
152 | * take place once the lock has been released via a store. |
153 | * |
154 | * The above two documents seem to contradict each other, |
155 | * however with the exception of early steppings of the Pentium |
156 | * Pro, the second document is closer to the truth: a store |
157 | * will always act as a load fence for all loads that precede |
158 | * the store in instruction order. |
159 | * |
160 | * Again, note that stores can be buffered and will not always |
161 | * become immediately visible to other CPUs: they are however |
162 | * buffered in order. |
163 | * |
164 | * AMD64 Stores occur in order and are buffered. Loads can be |
165 | * reordered, however stores act as load fences, meaning that |
166 | * loads can not be reordered around stores. |
167 | */ |
168 | SPINLOCK_INLINE void |
169 | __cpu_simple_unlock(__cpu_simple_lock_t *lockp) |
170 | { |
171 | |
172 | __insn_barrier(); |
173 | *lockp = __SIMPLELOCK_UNLOCKED; |
174 | } |
175 | |
176 | #endif /* SPINLOCK_BODY */ |
177 | |
178 | #endif /* _X86_LOCK_H_ */ |
179 | |