]>
Commit | Line | Data |
---|---|---|
eccb488f ER |
1 | diff -r 962aec0d731c innobase/configure |
2 | --- a/innobase/configure Thu Oct 09 08:28:53 2008 -0700 | |
3 | +++ b/innobase/configure Thu Oct 09 08:30:28 2008 -0700 | |
4 | @@ -20519,6 +20519,88 @@ | |
5 | ||
6 | fi | |
7 | done | |
8 | + | |
9 | + | |
10 | +# as http://lists.mysql.com/commits/40686 does | |
11 | +{ echo "$as_me:$LINENO: checking whether the compiler provides atomic builtins" >&5 | |
12 | +echo $ECHO_N "checking whether the compiler provides atomic builtins... $ECHO_C" >&6; } | |
13 | +if test "${mysql_cv_atomic_builtins+set}" = set; then | |
14 | + echo $ECHO_N "(cached) $ECHO_C" >&6 | |
15 | +else | |
16 | + if test "$cross_compiling" = yes; then | |
17 | + { { echo "$as_me:$LINENO: error: cannot run test program while cross compiling | |
18 | +See \`config.log' for more details." >&5 | |
19 | +echo "$as_me: error: cannot run test program while cross compiling | |
20 | +See \`config.log' for more details." >&2;} | |
21 | + { (exit 1); exit 1; }; } | |
22 | +else | |
23 | + cat >conftest.$ac_ext <<_ACEOF | |
24 | +/* confdefs.h. */ | |
25 | +_ACEOF | |
26 | +cat confdefs.h >>conftest.$ac_ext | |
27 | +cat >>conftest.$ac_ext <<_ACEOF | |
28 | +/* end confdefs.h. */ | |
29 | + | |
30 | + int main() | |
31 | + { | |
32 | + int foo= -10; int bar= 10; | |
33 | + __sync_fetch_and_add(&foo, bar); | |
34 | + if (foo) | |
35 | + return -1; | |
36 | + bar= __sync_lock_test_and_set(&foo, bar); | |
37 | + if (bar || foo != 10) | |
38 | + return -1; | |
39 | + bar= __sync_val_compare_and_swap(&bar, foo, 15); | |
40 | + if (bar) | |
41 | + return -1; | |
42 | + return 0; | |
43 | + } | |
44 | + | |
45 | +_ACEOF | |
46 | +rm -f conftest$ac_exeext | |
47 | +if { (ac_try="$ac_link" | |
48 | +case "(($ac_try" in | |
49 | + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; | |
50 | + *) ac_try_echo=$ac_try;; | |
51 | +esac | |
52 | +eval "echo \"\$as_me:$LINENO: $ac_try_echo\"") >&5 | |
53 | + (eval "$ac_link") 2>&5 | |
54 | + ac_status=$? | |
55 | + echo "$as_me:$LINENO: \$? = $ac_status" >&5 | |
56 | + (exit $ac_status); } && { ac_try='./conftest$ac_exeext' | |
57 | + { (case "(($ac_try" in | |
58 | + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; | |
59 | + *) ac_try_echo=$ac_try;; | |
60 | +esac | |
61 | +eval "echo \"\$as_me:$LINENO: $ac_try_echo\"") >&5 | |
62 | + (eval "$ac_try") 2>&5 | |
63 | + ac_status=$? | |
64 | + echo "$as_me:$LINENO: \$? = $ac_status" >&5 | |
65 | + (exit $ac_status); }; }; then | |
66 | + mysql_cv_atomic_builtins=yes | |
67 | +else | |
68 | + echo "$as_me: program exited with status $ac_status" >&5 | |
69 | +echo "$as_me: failed program was:" >&5 | |
70 | +sed 's/^/| /' conftest.$ac_ext >&5 | |
71 | + | |
72 | +( exit $ac_status ) | |
73 | +mysql_cv_atomic_builtins=no | |
74 | +fi | |
75 | +rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext conftest.$ac_objext conftest.$ac_ext | |
76 | +fi | |
77 | + | |
78 | + | |
79 | +fi | |
80 | +{ echo "$as_me:$LINENO: result: $mysql_cv_atomic_builtins" >&5 | |
81 | +echo "${ECHO_T}$mysql_cv_atomic_builtins" >&6; } | |
82 | + | |
83 | +if test "x$mysql_cv_atomic_builtins" = xyes; then | |
84 | + | |
85 | +cat >>confdefs.h <<\_ACEOF | |
86 | +#define HAVE_ATOMIC_BUILTINS 1 | |
87 | +_ACEOF | |
88 | + | |
89 | +fi | |
90 | ||
91 | #AC_CHECK_FUNCS(readdir_r) MySQL checks that it has also the right args. | |
92 | # Some versions of Unix only take 2 arguments. | |
93 | diff -r 962aec0d731c innobase/configure.in | |
94 | --- a/innobase/configure.in Thu Oct 09 08:28:53 2008 -0700 | |
95 | +++ b/innobase/configure.in Thu Oct 09 08:30:28 2008 -0700 | |
96 | @@ -42,6 +42,31 @@ | |
97 | AC_CHECK_FUNCS(sched_yield) | |
98 | AC_CHECK_FUNCS(fdatasync) | |
99 | AC_CHECK_FUNCS(localtime_r) | |
100 | + | |
101 | +# as http://lists.mysql.com/commits/40686 does | |
102 | +AC_CACHE_CHECK([whether the compiler provides atomic builtins], | |
103 | + [mysql_cv_atomic_builtins], [AC_TRY_RUN([ | |
104 | + int main() | |
105 | + { | |
106 | + int foo= -10; int bar= 10; | |
107 | + __sync_fetch_and_add(&foo, bar); | |
108 | + if (foo) | |
109 | + return -1; | |
110 | + bar= __sync_lock_test_and_set(&foo, bar); | |
111 | + if (bar || foo != 10) | |
112 | + return -1; | |
113 | + bar= __sync_val_compare_and_swap(&bar, foo, 15); | |
114 | + if (bar) | |
115 | + return -1; | |
116 | + return 0; | |
117 | + } | |
118 | +], [mysql_cv_atomic_builtins=yes], [mysql_cv_atomic_builtins=no])]) | |
119 | + | |
120 | +if test "x$mysql_cv_atomic_builtins" = xyes; then | |
121 | + AC_DEFINE(HAVE_ATOMIC_BUILTINS, 1, | |
122 | + [Define to 1 if compiler provides atomic builtins.]) | |
123 | +fi | |
124 | + | |
125 | #AC_CHECK_FUNCS(readdir_r) MySQL checks that it has also the right args. | |
126 | # Some versions of Unix only take 2 arguments. | |
127 | #AC_C_INLINE Already checked in MySQL | |
128 | diff -r 962aec0d731c innobase/ib_config.h | |
129 | --- a/innobase/ib_config.h Thu Oct 09 08:28:53 2008 -0700 | |
130 | +++ b/innobase/ib_config.h Thu Oct 09 08:30:28 2008 -0700 | |
131 | @@ -3,6 +3,9 @@ | |
132 | ||
133 | /* Define to 1 if you have the <aio.h> header file. */ | |
134 | #define HAVE_AIO_H 1 | |
135 | + | |
136 | +/* Define to 1 if compiler provides atomic builtins. */ | |
137 | +#define HAVE_ATOMIC_BUILTINS 1 | |
138 | ||
139 | /* Define to 1 if you have the <dlfcn.h> header file. */ | |
140 | #define HAVE_DLFCN_H 1 | |
141 | diff -r 962aec0d731c innobase/ib_config.h.in | |
142 | --- a/innobase/ib_config.h.in Thu Oct 09 08:28:53 2008 -0700 | |
143 | +++ b/innobase/ib_config.h.in Thu Oct 09 08:30:28 2008 -0700 | |
144 | @@ -2,6 +2,9 @@ | |
145 | ||
146 | /* Define to 1 if you have the <aio.h> header file. */ | |
147 | #undef HAVE_AIO_H | |
148 | + | |
149 | +/* Define to 1 if compiler provides atomic builtins. */ | |
150 | +#undef HAVE_ATOMIC_BUILTINS | |
151 | ||
152 | /* Define to 1 if you have the <dlfcn.h> header file. */ | |
153 | #undef HAVE_DLFCN_H | |
154 | diff -r 962aec0d731c innobase/include/sync0rw.h | |
155 | --- a/innobase/include/sync0rw.h Thu Oct 09 08:28:53 2008 -0700 | |
156 | +++ b/innobase/include/sync0rw.h Thu Oct 09 08:30:28 2008 -0700 | |
157 | @@ -325,7 +325,17 @@ | |
158 | Accessor functions for rw lock. */ | |
159 | UNIV_INLINE | |
160 | ulint | |
161 | -rw_lock_get_waiters( | |
162 | +rw_lock_get_s_waiters( | |
163 | +/*==================*/ | |
164 | + rw_lock_t* lock); | |
165 | +UNIV_INLINE | |
166 | +ulint | |
167 | +rw_lock_get_x_waiters( | |
168 | +/*==================*/ | |
169 | + rw_lock_t* lock); | |
170 | +UNIV_INLINE | |
171 | +ulint | |
172 | +rw_lock_get_wx_waiters( | |
173 | /*================*/ | |
174 | rw_lock_t* lock); | |
175 | UNIV_INLINE | |
176 | @@ -408,6 +418,11 @@ | |
177 | rw_lock_debug_t* info); /* in: debug struct */ | |
178 | #endif /* UNIV_SYNC_DEBUG */ | |
179 | ||
180 | +#ifdef HAVE_ATOMIC_BUILTINS | |
181 | +/* This value means NOT_LOCKED */ | |
182 | +#define RW_LOCK_BIAS 0x00100000 | |
183 | +#endif | |
184 | + | |
185 | /* NOTE! The structure appears here only for the compiler to know its size. | |
186 | Do not use its fields directly! The structure used in the spin lock | |
187 | implementation of a read-write lock. Several threads may have a shared lock | |
188 | @@ -417,9 +432,9 @@ | |
189 | field. Then no new readers are allowed in. */ | |
190 | ||
191 | struct rw_lock_struct { | |
192 | - os_event_t event; /* Used by sync0arr.c for thread queueing */ | |
193 | - | |
194 | -#ifdef __WIN__ | |
195 | + /* Used by sync0arr.c for thread queueing */ | |
196 | + os_event_t s_event; /* Used for s_lock */ | |
197 | + os_event_t x_event; /* Used for x_lock */ | |
198 | os_event_t wait_ex_event; /* This windows specific event is | |
199 | used by the thread which has set the | |
200 | lock state to RW_LOCK_WAIT_EX. The | |
201 | @@ -427,31 +442,35 @@ | |
202 | thread will be the next one to proceed | |
203 | once the current the event gets | |
204 | signalled. See LEMMA 2 in sync0sync.c */ | |
205 | + | |
206 | +#ifdef HAVE_ATOMIC_BUILTINS | |
207 | + volatile lint lock_word; /* Used by using atomic builtin */ | |
208 | #endif | |
209 | ||
210 | - ulint reader_count; /* Number of readers who have locked this | |
211 | + volatile ulint reader_count; /* Number of readers who have locked this | |
212 | lock in the shared mode */ | |
213 | - ulint writer; /* This field is set to RW_LOCK_EX if there | |
214 | + volatile ulint writer; /* This field is set to RW_LOCK_EX if there | |
215 | is a writer owning the lock (in exclusive | |
216 | mode), RW_LOCK_WAIT_EX if a writer is | |
217 | queueing for the lock, and | |
218 | RW_LOCK_NOT_LOCKED, otherwise. */ | |
219 | - os_thread_id_t writer_thread; | |
220 | + volatile os_thread_id_t writer_thread; | |
221 | /* Thread id of a possible writer thread */ | |
222 | - ulint writer_count; /* Number of times the same thread has | |
223 | + volatile ulint writer_count; /* Number of times the same thread has | |
224 | recursively locked the lock in the exclusive | |
225 | mode */ | |
226 | +#ifndef HAVE_ATOMIC_BUILTINS | |
227 | mutex_t mutex; /* The mutex protecting rw_lock_struct */ | |
228 | +#endif | |
229 | ulint pass; /* Default value 0. This is set to some | |
230 | value != 0 given by the caller of an x-lock | |
231 | operation, if the x-lock is to be passed to | |
232 | another thread to unlock (which happens in | |
233 | asynchronous i/o). */ | |
234 | - ulint waiters; /* This ulint is set to 1 if there are | |
235 | - waiters (readers or writers) in the global | |
236 | - wait array, waiting for this rw_lock. | |
237 | - Otherwise, == 0. */ | |
238 | - ibool writer_is_wait_ex; | |
239 | + volatile ulint s_waiters; /* 1: there are waiters (s_lock) */ | |
240 | + volatile ulint x_waiters; /* 1: there are waiters (x_lock) */ | |
241 | + volatile ulint wait_ex_waiters; /* 1: there are waiters (wait_ex) */ | |
242 | + volatile ibool writer_is_wait_ex; | |
243 | /* This is TRUE if the writer field is | |
244 | RW_LOCK_WAIT_EX; this field is located far | |
245 | from the memory update hotspot fields which | |
246 | diff -r 962aec0d731c innobase/include/sync0rw.ic | |
247 | --- a/innobase/include/sync0rw.ic Thu Oct 09 08:28:53 2008 -0700 | |
248 | +++ b/innobase/include/sync0rw.ic Thu Oct 09 08:30:28 2008 -0700 | |
249 | @@ -47,20 +47,52 @@ | |
250 | Accessor functions for rw lock. */ | |
251 | UNIV_INLINE | |
252 | ulint | |
253 | -rw_lock_get_waiters( | |
254 | +rw_lock_get_s_waiters( | |
255 | /*================*/ | |
256 | rw_lock_t* lock) | |
257 | { | |
258 | - return(lock->waiters); | |
259 | + return(lock->s_waiters); | |
260 | +} | |
261 | +UNIV_INLINE | |
262 | +ulint | |
263 | +rw_lock_get_x_waiters( | |
264 | +/*================*/ | |
265 | + rw_lock_t* lock) | |
266 | +{ | |
267 | + return(lock->x_waiters); | |
268 | +} | |
269 | +UNIV_INLINE | |
270 | +ulint | |
271 | +rw_lock_get_wx_waiters( | |
272 | +/*================*/ | |
273 | + rw_lock_t* lock) | |
274 | +{ | |
275 | + return(lock->wait_ex_waiters); | |
276 | } | |
277 | UNIV_INLINE | |
278 | void | |
279 | -rw_lock_set_waiters( | |
280 | -/*================*/ | |
281 | +rw_lock_set_s_waiters( | |
282 | rw_lock_t* lock, | |
283 | ulint flag) | |
284 | { | |
285 | - lock->waiters = flag; | |
286 | + lock->s_waiters = flag; | |
287 | +} | |
288 | +UNIV_INLINE | |
289 | +void | |
290 | +rw_lock_set_x_waiters( | |
291 | + rw_lock_t* lock, | |
292 | + ulint flag) | |
293 | +{ | |
294 | + lock->x_waiters = flag; | |
295 | +} | |
296 | +UNIV_INLINE | |
297 | +void | |
298 | +rw_lock_set_wx_waiters( | |
299 | +/*================*/ | |
300 | + rw_lock_t* lock, | |
301 | + ulint flag) | |
302 | +{ | |
303 | + lock->wait_ex_waiters = flag; | |
304 | } | |
305 | UNIV_INLINE | |
306 | ulint | |
307 | @@ -68,7 +100,19 @@ | |
308 | /*===============*/ | |
309 | rw_lock_t* lock) | |
310 | { | |
311 | +#ifdef HAVE_ATOMIC_BUILTINS | |
312 | + if (lock->writer == RW_LOCK_NOT_LOCKED) { | |
313 | + return(RW_LOCK_NOT_LOCKED); | |
314 | + } | |
315 | + | |
316 | + if (lock->writer_is_wait_ex) { | |
317 | + return(RW_LOCK_WAIT_EX); | |
318 | + } else { | |
319 | + return(RW_LOCK_EX); | |
320 | + } | |
321 | +#else | |
322 | return(lock->writer); | |
323 | +#endif | |
324 | } | |
325 | UNIV_INLINE | |
326 | void | |
327 | @@ -96,6 +140,7 @@ | |
328 | { | |
329 | lock->reader_count = count; | |
330 | } | |
331 | +#ifndef HAVE_ATOMIC_BUILTINS | |
332 | UNIV_INLINE | |
333 | mutex_t* | |
334 | rw_lock_get_mutex( | |
335 | @@ -104,6 +149,7 @@ | |
336 | { | |
337 | return(&(lock->mutex)); | |
338 | } | |
339 | +#endif | |
340 | ||
341 | /********************************************************************** | |
342 | Returns the value of writer_count for the lock. Does not reserve the lock | |
343 | @@ -133,14 +179,26 @@ | |
344 | const char* file_name, /* in: file name where lock requested */ | |
345 | ulint line) /* in: line where requested */ | |
346 | { | |
347 | -#ifdef UNIV_SYNC_DEBUG | |
348 | +#if defined(UNIV_SYNC_DEBUG) && !defined(HAVE_ATOMIC_BUILTINS) | |
349 | ut_ad(mutex_own(rw_lock_get_mutex(lock))); | |
350 | #endif /* UNIV_SYNC_DEBUG */ | |
351 | /* Check if the writer field is free */ | |
352 | ||
353 | +#ifdef HAVE_ATOMIC_BUILTINS | |
354 | + if (UNIV_LIKELY(rw_lock_get_writer(lock) == RW_LOCK_NOT_LOCKED)) { | |
355 | + /* try s-lock */ | |
356 | + if(__sync_sub_and_fetch(&(lock->lock_word),1) <= 0) { | |
357 | + /* fail */ | |
358 | + __sync_fetch_and_add(&(lock->lock_word),1); | |
359 | + return(FALSE); /* locking did not succeed */ | |
360 | + } | |
361 | + /* success */ | |
362 | + __sync_fetch_and_add(&(lock->reader_count),1); | |
363 | +#else | |
364 | if (UNIV_LIKELY(lock->writer == RW_LOCK_NOT_LOCKED)) { | |
365 | /* Set the shared lock by incrementing the reader count */ | |
366 | lock->reader_count++; | |
367 | +#endif | |
368 | ||
369 | #ifdef UNIV_SYNC_DEBUG | |
370 | rw_lock_add_debug_info(lock, pass, RW_LOCK_SHARED, file_name, | |
371 | @@ -167,11 +225,15 @@ | |
372 | const char* file_name, /* in: file name where requested */ | |
373 | ulint line) /* in: line where lock requested */ | |
374 | { | |
375 | - ut_ad(lock->writer == RW_LOCK_NOT_LOCKED); | |
376 | + ut_ad(rw_lock_get_writer(lock) == RW_LOCK_NOT_LOCKED); | |
377 | ut_ad(rw_lock_get_reader_count(lock) == 0); | |
378 | ||
379 | /* Set the shared lock by incrementing the reader count */ | |
380 | +#ifdef HAVE_ATOMIC_BUILTINS | |
381 | + __sync_fetch_and_add(&(lock->reader_count),1); | |
382 | +#else | |
383 | lock->reader_count++; | |
384 | +#endif | |
385 | ||
386 | lock->last_s_file_name = file_name; | |
387 | lock->last_s_line = line; | |
388 | @@ -199,7 +261,11 @@ | |
389 | ||
390 | rw_lock_set_writer(lock, RW_LOCK_EX); | |
391 | lock->writer_thread = os_thread_get_curr_id(); | |
392 | +#ifdef HAVE_ATOMIC_BUILTINS | |
393 | + __sync_fetch_and_add(&(lock->writer_count),1); | |
394 | +#else | |
395 | lock->writer_count++; | |
396 | +#endif | |
397 | lock->pass = 0; | |
398 | ||
399 | lock->last_x_file_name = file_name; | |
400 | @@ -241,15 +307,21 @@ | |
401 | ut_ad(!rw_lock_own(lock, RW_LOCK_SHARED)); /* see NOTE above */ | |
402 | #endif /* UNIV_SYNC_DEBUG */ | |
403 | ||
404 | +#ifndef HAVE_ATOMIC_BUILTINS | |
405 | mutex_enter(rw_lock_get_mutex(lock)); | |
406 | +#endif | |
407 | ||
408 | if (UNIV_LIKELY(rw_lock_s_lock_low(lock, pass, file_name, line))) { | |
409 | +#ifndef HAVE_ATOMIC_BUILTINS | |
410 | mutex_exit(rw_lock_get_mutex(lock)); | |
411 | +#endif | |
412 | ||
413 | return; /* Success */ | |
414 | } else { | |
415 | /* Did not succeed, try spin wait */ | |
416 | +#ifndef HAVE_ATOMIC_BUILTINS | |
417 | mutex_exit(rw_lock_get_mutex(lock)); | |
418 | +#endif | |
419 | ||
420 | rw_lock_s_lock_spin(lock, pass, file_name, line); | |
421 | ||
422 | @@ -272,11 +344,23 @@ | |
423 | { | |
424 | ibool success = FALSE; | |
425 | ||
426 | +#ifdef HAVE_ATOMIC_BUILTINS | |
427 | + if (rw_lock_get_writer(lock) == RW_LOCK_NOT_LOCKED) { | |
428 | + /* try s-lock */ | |
429 | + if(__sync_sub_and_fetch(&(lock->lock_word),1) <= 0) { | |
430 | + /* fail */ | |
431 | + __sync_fetch_and_add(&(lock->lock_word),1); | |
432 | + return(FALSE); /* locking did not succeed */ | |
433 | + } | |
434 | + /* success */ | |
435 | + __sync_fetch_and_add(&(lock->reader_count),1); | |
436 | +#else | |
437 | mutex_enter(rw_lock_get_mutex(lock)); | |
438 | ||
439 | if (lock->writer == RW_LOCK_NOT_LOCKED) { | |
440 | /* Set the shared lock by incrementing the reader count */ | |
441 | lock->reader_count++; | |
442 | +#endif | |
443 | ||
444 | #ifdef UNIV_SYNC_DEBUG | |
445 | rw_lock_add_debug_info(lock, 0, RW_LOCK_SHARED, file_name, | |
446 | @@ -289,7 +373,9 @@ | |
447 | success = TRUE; | |
448 | } | |
449 | ||
450 | +#ifndef HAVE_ATOMIC_BUILTINS | |
451 | mutex_exit(rw_lock_get_mutex(lock)); | |
452 | +#endif | |
453 | ||
454 | return(success); | |
455 | } | |
456 | @@ -309,6 +395,55 @@ | |
457 | { | |
458 | ibool success = FALSE; | |
459 | os_thread_id_t curr_thread = os_thread_get_curr_id(); | |
460 | +#ifdef HAVE_ATOMIC_BUILTINS | |
461 | + if ((lock->lock_word == RW_LOCK_BIAS) | |
462 | + && rw_lock_get_writer(lock) == RW_LOCK_NOT_LOCKED) { | |
463 | + /* try x-lock */ | |
464 | + if(__sync_sub_and_fetch(&(lock->lock_word), | |
465 | + RW_LOCK_BIAS) == 0) { | |
466 | + /* success */ | |
467 | + /* try to lock writer */ | |
468 | + if(__sync_lock_test_and_set(&(lock->writer),RW_LOCK_EX) | |
469 | + == RW_LOCK_NOT_LOCKED) { | |
470 | + /* success */ | |
471 | + lock->writer_thread = curr_thread; | |
472 | + lock->pass = 0; | |
473 | + lock->writer_is_wait_ex = FALSE; | |
474 | + /* next function may work as memory barrier */ | |
475 | + relock: | |
476 | + __sync_fetch_and_add(&(lock->writer_count),1); | |
477 | + | |
478 | +#ifdef UNIV_SYNC_DEBUG | |
479 | + rw_lock_add_debug_info(lock, 0, RW_LOCK_EX, file_name, line); | |
480 | +#endif | |
481 | + | |
482 | + lock->last_x_file_name = file_name; | |
483 | + lock->last_x_line = line; | |
484 | + | |
485 | + ut_ad(rw_lock_validate(lock)); | |
486 | + | |
487 | + return(TRUE); | |
488 | + } else { | |
489 | + /* x-unlock */ | |
490 | + __sync_fetch_and_add(&(lock->lock_word), | |
491 | + RW_LOCK_BIAS); | |
492 | + } | |
493 | + } else { | |
494 | + /* fail (x-lock) */ | |
495 | + __sync_fetch_and_add(&(lock->lock_word),RW_LOCK_BIAS); | |
496 | + } | |
497 | + } | |
498 | + | |
499 | + if (lock->pass == 0 | |
500 | + && os_thread_eq(lock->writer_thread, curr_thread) | |
501 | + && rw_lock_get_writer(lock) == RW_LOCK_EX) { | |
502 | + goto relock; | |
503 | + } | |
504 | + | |
505 | + ut_ad(rw_lock_validate(lock)); | |
506 | + | |
507 | + return(FALSE); | |
508 | +#else | |
509 | mutex_enter(rw_lock_get_mutex(lock)); | |
510 | ||
511 | if (UNIV_UNLIKELY(rw_lock_get_reader_count(lock) != 0)) { | |
512 | @@ -339,6 +474,7 @@ | |
513 | ut_ad(rw_lock_validate(lock)); | |
514 | ||
515 | return(success); | |
516 | +#endif | |
517 | } | |
518 | ||
519 | /********************************************************************** | |
520 | @@ -354,16 +490,33 @@ | |
521 | #endif | |
522 | ) | |
523 | { | |
524 | +#ifndef HAVE_ATOMIC_BUILTINS | |
525 | mutex_t* mutex = &(lock->mutex); | |
526 | - ibool sg = FALSE; | |
527 | +#endif | |
528 | + ibool x_sg = FALSE; | |
529 | + ibool wx_sg = FALSE; | |
530 | +#ifdef HAVE_ATOMIC_BUILTINS | |
531 | + ibool last = FALSE; | |
532 | +#endif | |
533 | ||
534 | +#ifndef HAVE_ATOMIC_BUILTINS | |
535 | /* Acquire the mutex protecting the rw-lock fields */ | |
536 | mutex_enter(mutex); | |
537 | +#endif | |
538 | ||
539 | /* Reset the shared lock by decrementing the reader count */ | |
540 | ||
541 | ut_a(lock->reader_count > 0); | |
542 | +#ifdef HAVE_ATOMIC_BUILTINS | |
543 | + /* unlock lock_word */ | |
544 | + __sync_fetch_and_add(&(lock->lock_word),1); | |
545 | + | |
546 | + if(__sync_sub_and_fetch(&(lock->reader_count),1) == 0) { | |
547 | + last = TRUE; | |
548 | + } | |
549 | +#else | |
550 | lock->reader_count--; | |
551 | +#endif | |
552 | ||
553 | #ifdef UNIV_SYNC_DEBUG | |
554 | rw_lock_remove_debug_info(lock, pass, RW_LOCK_SHARED); | |
555 | @@ -372,20 +525,36 @@ | |
556 | /* If there may be waiters and this was the last s-lock, | |
557 | signal the object */ | |
558 | ||
559 | - if (UNIV_UNLIKELY(lock->waiters) | |
560 | +#ifdef HAVE_ATOMIC_BUILTINS | |
561 | + if (UNIV_UNLIKELY(last && lock->wait_ex_waiters)) { | |
562 | +#else | |
563 | + if (UNIV_UNLIKELY(lock->wait_ex_waiters) | |
564 | && lock->reader_count == 0) { | |
565 | - sg = TRUE; | |
566 | +#endif | |
567 | + wx_sg = TRUE; | |
568 | ||
569 | - rw_lock_set_waiters(lock, 0); | |
570 | + rw_lock_set_wx_waiters(lock, 0); | |
571 | + } | |
572 | +#ifdef HAVE_ATOMIC_BUILTINS | |
573 | + else if (UNIV_UNLIKELY(last && lock->x_waiters)) { | |
574 | +#else | |
575 | + else if (UNIV_UNLIKELY(lock->x_waiters) | |
576 | + && lock->reader_count == 0) { | |
577 | +#endif | |
578 | + x_sg = TRUE; | |
579 | + | |
580 | + rw_lock_set_x_waiters(lock, 0); | |
581 | } | |
582 | ||
583 | +#ifndef HAVE_ATOMIC_BUILTINS | |
584 | mutex_exit(mutex); | |
585 | +#endif | |
586 | ||
587 | - if (UNIV_UNLIKELY(sg)) { | |
588 | -#ifdef __WIN__ | |
589 | + if (UNIV_UNLIKELY(wx_sg)) { | |
590 | os_event_set(lock->wait_ex_event); | |
591 | -#endif | |
592 | - os_event_set(lock->event); | |
593 | + sync_array_object_signalled(sync_primary_wait_array); | |
594 | + } else if (UNIV_UNLIKELY(x_sg)) { | |
595 | + os_event_set(lock->x_event); | |
596 | sync_array_object_signalled(sync_primary_wait_array); | |
597 | } | |
598 | ||
599 | @@ -409,13 +578,22 @@ | |
600 | ||
601 | ut_ad(lock->reader_count > 0); | |
602 | ||
603 | +#ifdef HAVE_ATOMIC_BUILTINS | |
604 | + __sync_sub_and_fetch(&(lock->reader_count),1); | |
605 | +#else | |
606 | lock->reader_count--; | |
607 | +#endif | |
608 | ||
609 | #ifdef UNIV_SYNC_DEBUG | |
610 | rw_lock_remove_debug_info(lock, 0, RW_LOCK_SHARED); | |
611 | #endif | |
612 | ||
613 | +#ifdef HAVE_ATOMIC_BUILTINS | |
614 | + ut_ad(!lock->s_waiters); | |
615 | + ut_ad(!lock->x_waiters); | |
616 | +#else | |
617 | ut_ad(!lock->waiters); | |
618 | +#endif | |
619 | ut_ad(rw_lock_validate(lock)); | |
620 | #ifdef UNIV_SYNC_PERF_STAT | |
621 | rw_s_exit_count++; | |
622 | @@ -435,41 +613,81 @@ | |
623 | #endif | |
624 | ) | |
625 | { | |
626 | - ibool sg = FALSE; | |
627 | +#ifdef HAVE_ATOMIC_BUILTINS | |
628 | + ibool last = FALSE; | |
629 | +#endif | |
630 | + ibool s_sg = FALSE; | |
631 | + ibool x_sg = FALSE; | |
632 | ||
633 | +#ifndef HAVE_ATOMIC_BUILTINS | |
634 | /* Acquire the mutex protecting the rw-lock fields */ | |
635 | mutex_enter(&(lock->mutex)); | |
636 | +#endif | |
637 | ||
638 | /* Reset the exclusive lock if this thread no longer has an x-mode | |
639 | lock */ | |
640 | ||
641 | ut_ad(lock->writer_count > 0); | |
642 | ||
643 | +#ifdef HAVE_ATOMIC_BUILTINS | |
644 | + if(__sync_sub_and_fetch(&(lock->writer_count),1) == 0) { | |
645 | + last = TRUE; | |
646 | + } | |
647 | + | |
648 | + if (last) { | |
649 | + /* unlock lock_word */ | |
650 | + __sync_fetch_and_add(&(lock->lock_word),RW_LOCK_BIAS); | |
651 | + | |
652 | + /* FIXME: It is a value of bad manners for pthread. | |
653 | + But we shouldn't keep an ID of not-owner. */ | |
654 | + lock->writer_thread = -1; | |
655 | + | |
656 | + /* atomic operation may be safer about memory order. */ | |
657 | + rw_lock_set_writer(lock, RW_LOCK_NOT_LOCKED); | |
658 | + __sync_synchronize(); | |
659 | + } | |
660 | +#else | |
661 | lock->writer_count--; | |
662 | ||
663 | if (lock->writer_count == 0) { | |
664 | rw_lock_set_writer(lock, RW_LOCK_NOT_LOCKED); | |
665 | } | |
666 | +#endif | |
667 | ||
668 | #ifdef UNIV_SYNC_DEBUG | |
669 | rw_lock_remove_debug_info(lock, pass, RW_LOCK_EX); | |
670 | #endif | |
671 | ||
672 | /* If there may be waiters, signal the lock */ | |
673 | - if (UNIV_UNLIKELY(lock->waiters) | |
674 | - && lock->writer_count == 0) { | |
675 | - | |
676 | - sg = TRUE; | |
677 | - rw_lock_set_waiters(lock, 0); | |
678 | +#ifdef HAVE_ATOMIC_BUILTINS | |
679 | + if (last) { | |
680 | +#else | |
681 | + if (lock->writer_count == 0) { | |
682 | +#endif | |
683 | + if(lock->s_waiters){ | |
684 | + s_sg = TRUE; | |
685 | + rw_lock_set_s_waiters(lock, 0); | |
686 | + } | |
687 | + if(lock->x_waiters){ | |
688 | + x_sg = TRUE; | |
689 | + rw_lock_set_x_waiters(lock, 0); | |
690 | + } | |
691 | } | |
692 | ||
693 | +#ifndef HAVE_ATOMIC_BUILTINS | |
694 | mutex_exit(&(lock->mutex)); | |
695 | +#endif | |
696 | ||
697 | - if (UNIV_UNLIKELY(sg)) { | |
698 | + if (UNIV_UNLIKELY(s_sg)) { | |
699 | + os_event_set(lock->s_event); | |
700 | + sync_array_object_signalled(sync_primary_wait_array); | |
701 | + } | |
702 | + if (UNIV_UNLIKELY(x_sg)) { | |
703 | #ifdef __WIN__ | |
704 | + /* I doubt the necessity of it. */ | |
705 | os_event_set(lock->wait_ex_event); | |
706 | #endif | |
707 | - os_event_set(lock->event); | |
708 | + os_event_set(lock->x_event); | |
709 | sync_array_object_signalled(sync_primary_wait_array); | |
710 | } | |
711 | ||
712 | @@ -494,9 +712,13 @@ | |
713 | ||
714 | ut_ad(lock->writer_count > 0); | |
715 | ||
716 | +#ifdef HAVE_ATOMIC_BUILTINS | |
717 | + if(__sync_sub_and_fetch(&(lock->writer_count),1) == 0) { | |
718 | +#else | |
719 | lock->writer_count--; | |
720 | ||
721 | if (lock->writer_count == 0) { | |
722 | +#endif | |
723 | rw_lock_set_writer(lock, RW_LOCK_NOT_LOCKED); | |
724 | } | |
725 | ||
726 | @@ -504,7 +726,12 @@ | |
727 | rw_lock_remove_debug_info(lock, 0, RW_LOCK_EX); | |
728 | #endif | |
729 | ||
730 | +#ifdef HAVE_ATOMIC_BUILTINS | |
731 | + ut_ad(!lock->s_waiters); | |
732 | + ut_ad(!lock->x_waiters); | |
733 | +#else | |
734 | ut_ad(!lock->waiters); | |
735 | +#endif | |
736 | ut_ad(rw_lock_validate(lock)); | |
737 | ||
738 | #ifdef UNIV_SYNC_PERF_STAT | |
739 | diff -r 962aec0d731c innobase/sync/sync0arr.c | |
740 | --- a/innobase/sync/sync0arr.c Thu Oct 09 08:28:53 2008 -0700 | |
741 | +++ b/innobase/sync/sync0arr.c Thu Oct 09 08:30:28 2008 -0700 | |
742 | @@ -309,13 +309,13 @@ | |
743 | { | |
744 | if (type == SYNC_MUTEX) { | |
745 | return(os_event_reset(((mutex_t *) object)->event)); | |
746 | -#ifdef __WIN__ | |
747 | } else if (type == RW_LOCK_WAIT_EX) { | |
748 | return(os_event_reset( | |
749 | ((rw_lock_t *) object)->wait_ex_event)); | |
750 | -#endif | |
751 | - } else { | |
752 | - return(os_event_reset(((rw_lock_t *) object)->event)); | |
753 | + } else if (type == RW_LOCK_SHARED) { | |
754 | + return(os_event_reset(((rw_lock_t *) object)->s_event)); | |
755 | + } else { /* RW_LOCK_EX */ | |
756 | + return(os_event_reset(((rw_lock_t *) object)->x_event)); | |
757 | } | |
758 | } | |
759 | ||
760 | @@ -415,15 +415,12 @@ | |
761 | ||
762 | if (cell->request_type == SYNC_MUTEX) { | |
763 | event = ((mutex_t*) cell->wait_object)->event; | |
764 | -#ifdef __WIN__ | |
765 | - /* On windows if the thread about to wait is the one which | |
766 | - has set the state of the rw_lock to RW_LOCK_WAIT_EX, then | |
767 | - it waits on a special event i.e.: wait_ex_event. */ | |
768 | } else if (cell->request_type == RW_LOCK_WAIT_EX) { | |
769 | event = ((rw_lock_t*) cell->wait_object)->wait_ex_event; | |
770 | -#endif | |
771 | - } else { | |
772 | - event = ((rw_lock_t*) cell->wait_object)->event; | |
773 | + } else if (cell->request_type == RW_LOCK_SHARED) { | |
774 | + event = ((rw_lock_t*) cell->wait_object)->s_event; | |
775 | + } else { | |
776 | + event = ((rw_lock_t*) cell->wait_object)->x_event; | |
777 | } | |
778 | ||
779 | cell->waiting = TRUE; | |
780 | @@ -464,6 +461,7 @@ | |
781 | mutex_t* mutex; | |
782 | rw_lock_t* rwlock; | |
783 | ulint type; | |
784 | + ulint writer; | |
785 | ||
786 | type = cell->request_type; | |
787 | ||
788 | @@ -492,12 +490,10 @@ | |
789 | (ulong) mutex->waiters); | |
790 | ||
791 | } else if (type == RW_LOCK_EX | |
792 | -#ifdef __WIN__ | |
793 | || type == RW_LOCK_WAIT_EX | |
794 | -#endif | |
795 | || type == RW_LOCK_SHARED) { | |
796 | ||
797 | - fputs(type == RW_LOCK_EX ? "X-lock on" : "S-lock on", file); | |
798 | + fputs(type == RW_LOCK_SHARED ? "S-lock on" : "X-lock on", file); | |
799 | ||
800 | rwlock = cell->old_wait_rw_lock; | |
801 | ||
802 | @@ -505,21 +501,23 @@ | |
803 | " RW-latch at %p created in file %s line %lu\n", | |
804 | rwlock, rwlock->cfile_name, | |
805 | (ulong) rwlock->cline); | |
806 | - if (rwlock->writer != RW_LOCK_NOT_LOCKED) { | |
807 | + writer = rw_lock_get_writer(rwlock); | |
808 | + if (writer != RW_LOCK_NOT_LOCKED) { | |
809 | fprintf(file, | |
810 | "a writer (thread id %lu) has reserved it in mode %s", | |
811 | (ulong) os_thread_pf(rwlock->writer_thread), | |
812 | - rwlock->writer == RW_LOCK_EX | |
813 | + writer == RW_LOCK_EX | |
814 | ? " exclusive\n" | |
815 | : " wait exclusive\n"); | |
816 | } | |
817 | ||
818 | fprintf(file, | |
819 | - "number of readers %lu, waiters flag %lu\n" | |
820 | + "number of readers %lu, s_waiters flag %lu, x_waiters flag %lu\n" | |
821 | "Last time read locked in file %s line %lu\n" | |
822 | "Last time write locked in file %s line %lu\n", | |
823 | (ulong) rwlock->reader_count, | |
824 | - (ulong) rwlock->waiters, | |
825 | + (ulong) rwlock->s_waiters, | |
826 | + (ulong) (rwlock->x_waiters || rwlock->wait_ex_waiters), | |
827 | rwlock->last_s_file_name, | |
828 | (ulong) rwlock->last_s_line, | |
829 | rwlock->last_x_file_name, | |
830 | @@ -839,11 +837,15 @@ | |
831 | /*========================*/ | |
832 | sync_array_t* arr) /* in: wait array */ | |
833 | { | |
834 | +#ifdef HAVE_ATOMIC_BUILTINS | |
835 | + __sync_fetch_and_add(&(arr->sg_count),1); | |
836 | +#else | |
837 | sync_array_enter(arr); | |
838 | ||
839 | arr->sg_count++; | |
840 | ||
841 | sync_array_exit(arr); | |
842 | +#endif | |
843 | } | |
844 | ||
845 | /************************************************************************** | |
846 | @@ -880,19 +882,23 @@ | |
847 | ||
848 | mutex = cell->wait_object; | |
849 | os_event_set(mutex->event); | |
850 | -#ifdef __WIN__ | |
851 | } else if (cell->request_type | |
852 | == RW_LOCK_WAIT_EX) { | |
853 | rw_lock_t* lock; | |
854 | ||
855 | lock = cell->wait_object; | |
856 | os_event_set(lock->wait_ex_event); | |
857 | -#endif | |
858 | - } else { | |
859 | + } else if (cell->request_type | |
860 | + == RW_LOCK_SHARED) { | |
861 | rw_lock_t* lock; | |
862 | ||
863 | lock = cell->wait_object; | |
864 | - os_event_set(lock->event); | |
865 | + os_event_set(lock->s_event); | |
866 | + } else { | |
867 | + rw_lock_t* lock; | |
868 | + | |
869 | + lock = cell->wait_object; | |
870 | + os_event_set(lock->x_event); | |
871 | } | |
872 | } | |
873 | } | |
874 | diff -r 962aec0d731c innobase/sync/sync0rw.c | |
875 | --- a/innobase/sync/sync0rw.c Thu Oct 09 08:28:53 2008 -0700 | |
876 | +++ b/innobase/sync/sync0rw.c Thu Oct 09 08:30:28 2008 -0700 | |
877 | @@ -99,6 +99,7 @@ | |
878 | object is created, then the following call initializes | |
879 | the sync system. */ | |
880 | ||
881 | +#ifndef HAVE_ATOMIC_BUILTINS | |
882 | mutex_create(rw_lock_get_mutex(lock)); | |
883 | mutex_set_level(rw_lock_get_mutex(lock), SYNC_NO_ORDER_CHECK); | |
884 | ||
885 | @@ -108,8 +109,14 @@ | |
886 | lock->mutex.cmutex_name = cmutex_name; | |
887 | lock->mutex.mutex_type = 1; | |
888 | #endif /* UNIV_DEBUG && !UNIV_HOTBACKUP */ | |
889 | +#endif /* !HAVE_ATOMIC_BUILTINS */ | |
890 | ||
891 | - rw_lock_set_waiters(lock, 0); | |
892 | +#ifdef HAVE_ATOMIC_BUILTINS | |
893 | + lock->lock_word = RW_LOCK_BIAS; | |
894 | +#endif | |
895 | + rw_lock_set_s_waiters(lock, 0); | |
896 | + rw_lock_set_x_waiters(lock, 0); | |
897 | + rw_lock_set_wx_waiters(lock, 0); | |
898 | rw_lock_set_writer(lock, RW_LOCK_NOT_LOCKED); | |
899 | lock->writer_count = 0; | |
900 | rw_lock_set_reader_count(lock, 0); | |
901 | @@ -130,11 +137,9 @@ | |
902 | lock->last_x_file_name = "not yet reserved"; | |
903 | lock->last_s_line = 0; | |
904 | lock->last_x_line = 0; | |
905 | - lock->event = os_event_create(NULL); | |
906 | - | |
907 | -#ifdef __WIN__ | |
908 | + lock->s_event = os_event_create(NULL); | |
909 | + lock->x_event = os_event_create(NULL); | |
910 | lock->wait_ex_event = os_event_create(NULL); | |
911 | -#endif | |
912 | ||
913 | mutex_enter(&rw_lock_list_mutex); | |
914 | ||
915 | @@ -162,19 +167,21 @@ | |
916 | ut_a(rw_lock_validate(lock)); | |
917 | #endif /* UNIV_DEBUG */ | |
918 | ut_a(rw_lock_get_writer(lock) == RW_LOCK_NOT_LOCKED); | |
919 | - ut_a(rw_lock_get_waiters(lock) == 0); | |
920 | + ut_a(rw_lock_get_s_waiters(lock) == 0); | |
921 | + ut_a(rw_lock_get_x_waiters(lock) == 0); | |
922 | + ut_a(rw_lock_get_wx_waiters(lock) == 0); | |
923 | ut_a(rw_lock_get_reader_count(lock) == 0); | |
924 | ||
925 | lock->magic_n = 0; | |
926 | ||
927 | +#ifndef HAVE_ATOMIC_BUILTINS | |
928 | mutex_free(rw_lock_get_mutex(lock)); | |
929 | +#endif | |
930 | ||
931 | mutex_enter(&rw_lock_list_mutex); | |
932 | - os_event_free(lock->event); | |
933 | - | |
934 | -#ifdef __WIN__ | |
935 | + os_event_free(lock->s_event); | |
936 | + os_event_free(lock->x_event); | |
937 | os_event_free(lock->wait_ex_event); | |
938 | -#endif | |
939 | ||
940 | if (UT_LIST_GET_PREV(list, lock)) { | |
941 | ut_a(UT_LIST_GET_PREV(list, lock)->magic_n == RW_LOCK_MAGIC_N); | |
942 | @@ -192,6 +199,8 @@ | |
943 | Checks that the rw-lock has been initialized and that there are no | |
944 | simultaneous shared and exclusive locks. */ | |
945 | ||
946 | +/* MEMO: If HAVE_ATOMIC_BUILTINS, we should use this function statically. */ | |
947 | + | |
948 | ibool | |
949 | rw_lock_validate( | |
950 | /*=============*/ | |
951 | @@ -199,7 +208,9 @@ | |
952 | { | |
953 | ut_a(lock); | |
954 | ||
955 | +#ifndef HAVE_ATOMIC_BUILTINS | |
956 | mutex_enter(rw_lock_get_mutex(lock)); | |
957 | +#endif | |
958 | ||
959 | ut_a(lock->magic_n == RW_LOCK_MAGIC_N); | |
960 | ut_a((rw_lock_get_reader_count(lock) == 0) | |
961 | @@ -207,11 +218,17 @@ | |
962 | ut_a((rw_lock_get_writer(lock) == RW_LOCK_EX) | |
963 | || (rw_lock_get_writer(lock) == RW_LOCK_WAIT_EX) | |
964 | || (rw_lock_get_writer(lock) == RW_LOCK_NOT_LOCKED)); | |
965 | - ut_a((rw_lock_get_waiters(lock) == 0) | |
966 | - || (rw_lock_get_waiters(lock) == 1)); | |
967 | + ut_a((rw_lock_get_s_waiters(lock) == 0) | |
968 | + || (rw_lock_get_s_waiters(lock) == 1)); | |
969 | + ut_a((rw_lock_get_x_waiters(lock) == 0) | |
970 | + || (rw_lock_get_x_waiters(lock) == 1)); | |
971 | + ut_a((rw_lock_get_wx_waiters(lock) == 0) | |
972 | + || (rw_lock_get_wx_waiters(lock) == 1)); | |
973 | ut_a((lock->writer != RW_LOCK_EX) || (lock->writer_count > 0)); | |
974 | ||
975 | +#ifndef HAVE_ATOMIC_BUILTINS | |
976 | mutex_exit(rw_lock_get_mutex(lock)); | |
977 | +#endif | |
978 | ||
979 | return(TRUE); | |
980 | } | |
981 | @@ -237,13 +254,14 @@ | |
982 | ut_ad(rw_lock_validate(lock)); | |
983 | ||
984 | lock_loop: | |
985 | + i = 0; | |
986 | +spin_loop: | |
987 | rw_s_spin_wait_count++; | |
988 | ||
989 | /* Spin waiting for the writer field to become free */ | |
990 | - i = 0; | |
991 | ||
992 | - while (rw_lock_get_writer(lock) != RW_LOCK_NOT_LOCKED | |
993 | - && i < SYNC_SPIN_ROUNDS) { | |
994 | + while (i < SYNC_SPIN_ROUNDS | |
995 | + && rw_lock_get_writer(lock) != RW_LOCK_NOT_LOCKED) { | |
996 | if (srv_spin_wait_delay) { | |
997 | ut_delay(ut_rnd_interval(0, srv_spin_wait_delay)); | |
998 | } | |
999 | @@ -262,15 +280,27 @@ | |
1000 | lock->cfile_name, (ulong) lock->cline, (ulong) i); | |
1001 | } | |
1002 | ||
1003 | +#ifndef HAVE_ATOMIC_BUILTINS | |
1004 | mutex_enter(rw_lock_get_mutex(lock)); | |
1005 | +#endif | |
1006 | ||
1007 | /* We try once again to obtain the lock */ | |
1008 | ||
1009 | if (TRUE == rw_lock_s_lock_low(lock, pass, file_name, line)) { | |
1010 | +#ifndef HAVE_ATOMIC_BUILTINS | |
1011 | mutex_exit(rw_lock_get_mutex(lock)); | |
1012 | +#endif | |
1013 | ||
1014 | return; /* Success */ | |
1015 | } else { | |
1016 | +#ifdef HAVE_ATOMIC_BUILTINS | |
1017 | + /* like sync0sync.c doing */ | |
1018 | + i++; | |
1019 | + | |
1020 | + if (i < SYNC_SPIN_ROUNDS) { | |
1021 | + goto spin_loop; | |
1022 | + } | |
1023 | +#endif | |
1024 | /* If we get here, locking did not succeed, we may | |
1025 | suspend the thread to wait in the wait array */ | |
1026 | ||
1027 | @@ -281,9 +311,19 @@ | |
1028 | file_name, line, | |
1029 | &index); | |
1030 | ||
1031 | - rw_lock_set_waiters(lock, 1); | |
1032 | + rw_lock_set_s_waiters(lock, 1); | |
1033 | ||
1034 | +#ifdef HAVE_ATOMIC_BUILTINS | |
1035 | + /* like sync0sync.c doing */ | |
1036 | + for (i = 0; i < 4; i++) { | |
1037 | + if (TRUE == rw_lock_s_lock_low(lock, pass, file_name, line)) { | |
1038 | + sync_array_free_cell(sync_primary_wait_array, index); | |
1039 | + return; /* Success */ | |
1040 | + } | |
1041 | + } | |
1042 | +#else | |
1043 | mutex_exit(rw_lock_get_mutex(lock)); | |
1044 | +#endif | |
1045 | ||
1046 | if (srv_print_latch_waits) { | |
1047 | fprintf(stderr, | |
1048 | @@ -318,13 +358,19 @@ | |
1049 | { | |
1050 | ut_ad(rw_lock_is_locked(lock, RW_LOCK_EX)); | |
1051 | ||
1052 | +#ifndef HAVE_ATOMIC_BUILTINS | |
1053 | mutex_enter(&(lock->mutex)); | |
1054 | +#endif | |
1055 | ||
1056 | lock->writer_thread = os_thread_get_curr_id(); | |
1057 | ||
1058 | lock->pass = 0; | |
1059 | ||
1060 | +#ifndef HAVE_ATOMIC_BUILTINS | |
1061 | mutex_exit(&(lock->mutex)); | |
1062 | +#else | |
1063 | + __sync_synchronize(); | |
1064 | +#endif | |
1065 | } | |
1066 | ||
1067 | /********************************************************************** | |
1068 | @@ -342,6 +388,89 @@ | |
1069 | const char* file_name,/* in: file name where lock requested */ | |
1070 | ulint line) /* in: line where requested */ | |
1071 | { | |
1072 | +#ifdef HAVE_ATOMIC_BUILTINS | |
1073 | + os_thread_id_t curr_thread = os_thread_get_curr_id(); | |
1074 | + | |
1075 | + /* try to lock writer */ | |
1076 | + if(__sync_lock_test_and_set(&(lock->writer),RW_LOCK_EX) | |
1077 | + == RW_LOCK_NOT_LOCKED) { | |
1078 | + /* success */ | |
1079 | + /* obtain RW_LOCK_WAIT_EX right */ | |
1080 | + lock->writer_thread = curr_thread; | |
1081 | + lock->pass = pass; | |
1082 | + lock->writer_is_wait_ex = TRUE; | |
1083 | + /* atomic operation may be safer about memory order. */ | |
1084 | + __sync_synchronize(); | |
1085 | +#ifdef UNIV_SYNC_DEBUG | |
1086 | + rw_lock_add_debug_info(lock, pass, RW_LOCK_WAIT_EX, | |
1087 | + file_name, line); | |
1088 | +#endif | |
1089 | + } | |
1090 | + | |
1091 | + if (!os_thread_eq(lock->writer_thread, curr_thread)) { | |
1092 | + return(RW_LOCK_NOT_LOCKED); | |
1093 | + } | |
1094 | + | |
1095 | + switch(rw_lock_get_writer(lock)) { | |
1096 | + case RW_LOCK_WAIT_EX: | |
1097 | + /* have right to try x-lock */ | |
1098 | + if (lock->lock_word == RW_LOCK_BIAS) { | |
1099 | + /* try x-lock */ | |
1100 | + if(__sync_sub_and_fetch(&(lock->lock_word), | |
1101 | + RW_LOCK_BIAS) == 0) { | |
1102 | + /* success */ | |
1103 | + lock->pass = pass; | |
1104 | + lock->writer_is_wait_ex = FALSE; | |
1105 | + __sync_fetch_and_add(&(lock->writer_count),1); | |
1106 | + | |
1107 | +#ifdef UNIV_SYNC_DEBUG | |
1108 | + rw_lock_remove_debug_info(lock, pass, RW_LOCK_WAIT_EX); | |
1109 | + rw_lock_add_debug_info(lock, pass, RW_LOCK_EX, | |
1110 | + file_name, line); | |
1111 | +#endif | |
1112 | + | |
1113 | + lock->last_x_file_name = file_name; | |
1114 | + lock->last_x_line = line; | |
1115 | + | |
1116 | + /* Locking succeeded, we may return */ | |
1117 | + return(RW_LOCK_EX); | |
1118 | + } else { | |
1119 | + /* fail */ | |
1120 | + __sync_fetch_and_add(&(lock->lock_word), | |
1121 | + RW_LOCK_BIAS); | |
1122 | + } | |
1123 | + } | |
1124 | + /* There are readers, we have to wait */ | |
1125 | + return(RW_LOCK_WAIT_EX); | |
1126 | + | |
1127 | + break; | |
1128 | + | |
1129 | + case RW_LOCK_EX: | |
1130 | + /* already have x-lock */ | |
1131 | + if ((lock->pass == 0)&&(pass == 0)) { | |
1132 | + __sync_fetch_and_add(&(lock->writer_count),1); | |
1133 | + | |
1134 | +#ifdef UNIV_SYNC_DEBUG | |
1135 | + rw_lock_add_debug_info(lock, pass, RW_LOCK_EX, file_name, | |
1136 | + line); | |
1137 | +#endif | |
1138 | + | |
1139 | + lock->last_x_file_name = file_name; | |
1140 | + lock->last_x_line = line; | |
1141 | + | |
1142 | + /* Locking succeeded, we may return */ | |
1143 | + return(RW_LOCK_EX); | |
1144 | + } | |
1145 | + | |
1146 | + return(RW_LOCK_NOT_LOCKED); | |
1147 | + | |
1148 | + break; | |
1149 | + | |
1150 | + default: /* ??? */ | |
1151 | + return(RW_LOCK_NOT_LOCKED); | |
1152 | + } | |
1153 | +#else /* HAVE_ATOMIC_BUILTINS */ | |
1154 | + | |
1155 | #ifdef UNIV_SYNC_DEBUG | |
1156 | ut_ad(mutex_own(rw_lock_get_mutex(lock))); | |
1157 | #endif /* UNIV_SYNC_DEBUG */ | |
1158 | @@ -423,6 +552,7 @@ | |
1159 | /* Locking succeeded, we may return */ | |
1160 | return(RW_LOCK_EX); | |
1161 | } | |
1162 | +#endif /* HAVE_ATOMIC_BUILTINS */ | |
1163 | ||
1164 | /* Locking did not succeed */ | |
1165 | return(RW_LOCK_NOT_LOCKED); | |
1166 | @@ -448,19 +578,33 @@ | |
1167 | ulint line) /* in: line where requested */ | |
1168 | { | |
1169 | ulint index; /* index of the reserved wait cell */ | |
1170 | - ulint state; /* lock state acquired */ | |
1171 | + ulint state = RW_LOCK_NOT_LOCKED; /* lock state acquired */ | |
1172 | +#ifdef HAVE_ATOMIC_BUILTINS | |
1173 | + ulint prev_state = RW_LOCK_NOT_LOCKED; | |
1174 | +#endif | |
1175 | ulint i; /* spin round count */ | |
1176 | ||
1177 | ut_ad(rw_lock_validate(lock)); | |
1178 | ||
1179 | lock_loop: | |
1180 | + i = 0; | |
1181 | + | |
1182 | +#ifdef HAVE_ATOMIC_BUILTINS | |
1183 | + prev_state = state; | |
1184 | +#else | |
1185 | /* Acquire the mutex protecting the rw-lock fields */ | |
1186 | mutex_enter_fast(&(lock->mutex)); | |
1187 | +#endif | |
1188 | ||
1189 | state = rw_lock_x_lock_low(lock, pass, file_name, line); | |
1190 | ||
1191 | +#ifdef HAVE_ATOMIC_BUILTINS | |
1192 | + if (state != prev_state) i=0; /* if progress, reset counter. */ | |
1193 | +#else | |
1194 | mutex_exit(&(lock->mutex)); | |
1195 | +#endif | |
1196 | ||
1197 | +spin_loop: | |
1198 | if (state == RW_LOCK_EX) { | |
1199 | ||
1200 | return; /* Locking succeeded */ | |
1201 | @@ -468,10 +612,9 @@ | |
1202 | } else if (state == RW_LOCK_NOT_LOCKED) { | |
1203 | ||
1204 | /* Spin waiting for the writer field to become free */ | |
1205 | - i = 0; | |
1206 | ||
1207 | - while (rw_lock_get_writer(lock) != RW_LOCK_NOT_LOCKED | |
1208 | - && i < SYNC_SPIN_ROUNDS) { | |
1209 | + while (i < SYNC_SPIN_ROUNDS | |
1210 | + && rw_lock_get_writer(lock) != RW_LOCK_NOT_LOCKED) { | |
1211 | if (srv_spin_wait_delay) { | |
1212 | ut_delay(ut_rnd_interval(0, | |
1213 | srv_spin_wait_delay)); | |
1214 | @@ -485,9 +628,12 @@ | |
1215 | } else if (state == RW_LOCK_WAIT_EX) { | |
1216 | ||
1217 | /* Spin waiting for the reader count field to become zero */ | |
1218 | - i = 0; | |
1219 | ||
1220 | +#ifdef HAVE_ATOMIC_BUILTINS | |
1221 | + while (lock->lock_word != RW_LOCK_BIAS | |
1222 | +#else | |
1223 | while (rw_lock_get_reader_count(lock) != 0 | |
1224 | +#endif | |
1225 | && i < SYNC_SPIN_ROUNDS) { | |
1226 | if (srv_spin_wait_delay) { | |
1227 | ut_delay(ut_rnd_interval(0, | |
1228 | @@ -500,7 +646,6 @@ | |
1229 | os_thread_yield(); | |
1230 | } | |
1231 | } else { | |
1232 | - i = 0; /* Eliminate a compiler warning */ | |
1233 | ut_error; | |
1234 | } | |
1235 | ||
1236 | @@ -516,34 +661,69 @@ | |
1237 | /* We try once again to obtain the lock. Acquire the mutex protecting | |
1238 | the rw-lock fields */ | |
1239 | ||
1240 | +#ifdef HAVE_ATOMIC_BUILTINS | |
1241 | + prev_state = state; | |
1242 | +#else | |
1243 | mutex_enter(rw_lock_get_mutex(lock)); | |
1244 | +#endif | |
1245 | ||
1246 | state = rw_lock_x_lock_low(lock, pass, file_name, line); | |
1247 | ||
1248 | +#ifdef HAVE_ATOMIC_BUILTINS | |
1249 | + if (state != prev_state) i=0; /* if progress, reset counter. */ | |
1250 | +#endif | |
1251 | + | |
1252 | if (state == RW_LOCK_EX) { | |
1253 | +#ifndef HAVE_ATOMIC_BUILTINS | |
1254 | mutex_exit(rw_lock_get_mutex(lock)); | |
1255 | +#endif | |
1256 | ||
1257 | return; /* Locking succeeded */ | |
1258 | } | |
1259 | + | |
1260 | +#ifdef HAVE_ATOMIC_BUILTINS | |
1261 | + /* like sync0sync.c doing */ | |
1262 | + i++; | |
1263 | + | |
1264 | + if (i < SYNC_SPIN_ROUNDS) { | |
1265 | + goto spin_loop; | |
1266 | + } | |
1267 | +#endif | |
1268 | ||
1269 | rw_x_system_call_count++; | |
1270 | ||
1271 | sync_array_reserve_cell(sync_primary_wait_array, | |
1272 | lock, | |
1273 | -#ifdef __WIN__ | |
1274 | - /* On windows RW_LOCK_WAIT_EX signifies | |
1275 | - that this thread should wait on the | |
1276 | - special wait_ex_event. */ | |
1277 | (state == RW_LOCK_WAIT_EX) | |
1278 | ? RW_LOCK_WAIT_EX : | |
1279 | -#endif | |
1280 | RW_LOCK_EX, | |
1281 | file_name, line, | |
1282 | &index); | |
1283 | ||
1284 | - rw_lock_set_waiters(lock, 1); | |
1285 | + if (state == RW_LOCK_WAIT_EX) { | |
1286 | + rw_lock_set_wx_waiters(lock, 1); | |
1287 | + } else { | |
1288 | + rw_lock_set_x_waiters(lock, 1); | |
1289 | + } | |
1290 | ||
1291 | +#ifdef HAVE_ATOMIC_BUILTINS | |
1292 | + /* like sync0sync.c doing */ | |
1293 | + for (i = 0; i < 4; i++) { | |
1294 | + prev_state = state; | |
1295 | + state = rw_lock_x_lock_low(lock, pass, file_name, line); | |
1296 | + if (state == RW_LOCK_EX) { | |
1297 | + sync_array_free_cell(sync_primary_wait_array, index); | |
1298 | + return; /* Locking succeeded */ | |
1299 | + } | |
1300 | + if (state != prev_state) { | |
1301 | + /* retry! */ | |
1302 | + sync_array_free_cell(sync_primary_wait_array, index); | |
1303 | + goto lock_loop; | |
1304 | + } | |
1305 | + } | |
1306 | +#else | |
1307 | mutex_exit(rw_lock_get_mutex(lock)); | |
1308 | +#endif | |
1309 | ||
1310 | if (srv_print_latch_waits) { | |
1311 | fprintf(stderr, | |
1312 | @@ -718,7 +898,9 @@ | |
1313 | ut_ad(lock); | |
1314 | ut_ad(rw_lock_validate(lock)); | |
1315 | ||
1316 | +#ifndef HAVE_ATOMIC_BUILTINS | |
1317 | mutex_enter(&(lock->mutex)); | |
1318 | +#endif | |
1319 | ||
1320 | info = UT_LIST_GET_FIRST(lock->debug_list); | |
1321 | ||
1322 | @@ -728,7 +910,9 @@ | |
1323 | && (info->pass == 0) | |
1324 | && (info->lock_type == lock_type)) { | |
1325 | ||
1326 | +#ifndef HAVE_ATOMIC_BUILTINS | |
1327 | mutex_exit(&(lock->mutex)); | |
1328 | +#endif | |
1329 | /* Found! */ | |
1330 | ||
1331 | return(TRUE); | |
1332 | @@ -736,7 +920,9 @@ | |
1333 | ||
1334 | info = UT_LIST_GET_NEXT(list, info); | |
1335 | } | |
1336 | +#ifndef HAVE_ATOMIC_BUILTINS | |
1337 | mutex_exit(&(lock->mutex)); | |
1338 | +#endif | |
1339 | ||
1340 | return(FALSE); | |
1341 | } | |
1342 | @@ -758,21 +944,25 @@ | |
1343 | ut_ad(lock); | |
1344 | ut_ad(rw_lock_validate(lock)); | |
1345 | ||
1346 | +#ifndef HAVE_ATOMIC_BUILTINS | |
1347 | mutex_enter(&(lock->mutex)); | |
1348 | +#endif | |
1349 | ||
1350 | if (lock_type == RW_LOCK_SHARED) { | |
1351 | if (lock->reader_count > 0) { | |
1352 | ret = TRUE; | |
1353 | } | |
1354 | } else if (lock_type == RW_LOCK_EX) { | |
1355 | - if (lock->writer == RW_LOCK_EX) { | |
1356 | + if (rw_lock_get_writer(lock) == RW_LOCK_EX) { | |
1357 | ret = TRUE; | |
1358 | } | |
1359 | } else { | |
1360 | ut_error; | |
1361 | } | |
1362 | ||
1363 | +#ifndef HAVE_ATOMIC_BUILTINS | |
1364 | mutex_exit(&(lock->mutex)); | |
1365 | +#endif | |
1366 | ||
1367 | return(ret); | |
1368 | } | |
1369 | @@ -801,16 +991,26 @@ | |
1370 | ||
1371 | count++; | |
1372 | ||
1373 | +#ifndef HAVE_ATOMIC_BUILTINS | |
1374 | mutex_enter(&(lock->mutex)); | |
1375 | +#endif | |
1376 | ||
1377 | if ((rw_lock_get_writer(lock) != RW_LOCK_NOT_LOCKED) | |
1378 | || (rw_lock_get_reader_count(lock) != 0) | |
1379 | - || (rw_lock_get_waiters(lock) != 0)) { | |
1380 | + || (rw_lock_get_s_waiters(lock) != 0) | |
1381 | + || (rw_lock_get_x_waiters(lock) != 0) | |
1382 | + || (rw_lock_get_wx_waiters(lock) != 0)) { | |
1383 | ||
1384 | fprintf(stderr, "RW-LOCK: %p ", lock); | |
1385 | ||
1386 | - if (rw_lock_get_waiters(lock)) { | |
1387 | - fputs(" Waiters for the lock exist\n", stderr); | |
1388 | + if (rw_lock_get_s_waiters(lock)) { | |
1389 | + fputs(" s_waiters for the lock exist,", stderr); | |
1390 | + } | |
1391 | + if (rw_lock_get_x_waiters(lock)) { | |
1392 | + fputs(" x_waiters for the lock exist\n", stderr); | |
1393 | + } | |
1394 | + if (rw_lock_get_wx_waiters(lock)) { | |
1395 | + fputs(" wait_ex_waiters for the lock exist\n", stderr); | |
1396 | } else { | |
1397 | putc('\n', stderr); | |
1398 | } | |
1399 | @@ -822,7 +1022,9 @@ | |
1400 | } | |
1401 | } | |
1402 | ||
1403 | +#ifndef HAVE_ATOMIC_BUILTINS | |
1404 | mutex_exit(&(lock->mutex)); | |
1405 | +#endif | |
1406 | lock = UT_LIST_GET_NEXT(list, lock); | |
1407 | } | |
1408 | ||
1409 | @@ -847,10 +1049,18 @@ | |
1410 | ||
1411 | if ((rw_lock_get_writer(lock) != RW_LOCK_NOT_LOCKED) | |
1412 | || (rw_lock_get_reader_count(lock) != 0) | |
1413 | - || (rw_lock_get_waiters(lock) != 0)) { | |
1414 | + || (rw_lock_get_s_waiters(lock) != 0) | |
1415 | + || (rw_lock_get_x_waiters(lock) != 0) | |
1416 | + || (rw_lock_get_wx_waiters(lock) != 0)) { | |
1417 | ||
1418 | - if (rw_lock_get_waiters(lock)) { | |
1419 | - fputs(" Waiters for the lock exist\n", stderr); | |
1420 | + if (rw_lock_get_s_waiters(lock)) { | |
1421 | + fputs(" s_waiters for the lock exist,", stderr); | |
1422 | + } | |
1423 | + if (rw_lock_get_x_waiters(lock)) { | |
1424 | + fputs(" x_waiters for the lock exist\n", stderr); | |
1425 | + } | |
1426 | + if (rw_lock_get_wx_waiters(lock)) { | |
1427 | + fputs(" wait_ex_waiters for the lock exist\n", stderr); | |
1428 | } else { | |
1429 | putc('\n', stderr); | |
1430 | } | |
1431 | @@ -909,14 +1119,18 @@ | |
1432 | lock = UT_LIST_GET_FIRST(rw_lock_list); | |
1433 | ||
1434 | while (lock != NULL) { | |
1435 | +#ifndef HAVE_ATOMIC_BUILTINS | |
1436 | mutex_enter(rw_lock_get_mutex(lock)); | |
1437 | +#endif | |
1438 | ||
1439 | if ((rw_lock_get_writer(lock) != RW_LOCK_NOT_LOCKED) | |
1440 | || (rw_lock_get_reader_count(lock) != 0)) { | |
1441 | count++; | |
1442 | } | |
1443 | ||
1444 | +#ifndef HAVE_ATOMIC_BUILTINS | |
1445 | mutex_exit(rw_lock_get_mutex(lock)); | |
1446 | +#endif | |
1447 | lock = UT_LIST_GET_NEXT(list, lock); | |
1448 | } | |
1449 | ||
1450 | diff -r 962aec0d731c patch_info/innodb_rw_lock.info | |
1451 | --- /dev/null Thu Jan 01 00:00:00 1970 +0000 | |
1452 | +++ b/patch_info/innodb_rw_lock.info Thu Oct 09 08:30:28 2008 -0700 | |
1453 | @@ -0,0 +1,6 @@ | |
1454 | +File=innodb_rw_lock.patch | |
1455 | +Name=Fix of InnoDB rw_locks | |
1456 | +Version=1.0 | |
1457 | +Author=Yasufumi Kinoshita | |
1458 | +License=BSD | |
1459 | +Comment= |