]>
Commit | Line | Data |
---|---|---|
45532174 ER |
1 | diff -ruN a/innobase/btr/btr0sea.c b/innobase/btr/btr0sea.c |
2 | --- a/innobase/btr/btr0sea.c 2009-05-20 14:21:44.000000000 +0900 | |
3 | +++ b/innobase/btr/btr0sea.c 2009-05-20 14:39:34.000000000 +0900 | |
4 | @@ -773,7 +773,7 @@ | |
5 | rw_lock_s_lock(&btr_search_latch); | |
6 | } | |
eccb488f | 7 | |
45532174 | 8 | - ut_ad(btr_search_latch.writer != RW_LOCK_EX); |
89b96684 | 9 | + ut_ad(btr_search_latch.writer_count == 0); |
45532174 ER |
10 | ut_ad(btr_search_latch.reader_count > 0); |
11 | ||
12 | rec = ha_search_and_get_data(btr_search_sys->hash_index, fold); | |
13 | diff -ruN a/innobase/include/sync0rw.h b/innobase/include/sync0rw.h | |
14 | --- a/innobase/include/sync0rw.h 2009-01-30 06:42:20.000000000 +0900 | |
15 | +++ b/innobase/include/sync0rw.h 2009-04-16 16:15:28.000000000 +0900 | |
eccb488f ER |
16 | @@ -325,7 +325,17 @@ |
17 | Accessor functions for rw lock. */ | |
18 | UNIV_INLINE | |
19 | ulint | |
20 | -rw_lock_get_waiters( | |
21 | +rw_lock_get_s_waiters( | |
22 | +/*==================*/ | |
23 | + rw_lock_t* lock); | |
24 | +UNIV_INLINE | |
25 | +ulint | |
26 | +rw_lock_get_x_waiters( | |
27 | +/*==================*/ | |
28 | + rw_lock_t* lock); | |
29 | +UNIV_INLINE | |
30 | +ulint | |
31 | +rw_lock_get_wx_waiters( | |
32 | /*================*/ | |
33 | rw_lock_t* lock); | |
34 | UNIV_INLINE | |
337bc045 | 35 | @@ -408,6 +418,17 @@ |
eccb488f ER |
36 | rw_lock_debug_t* info); /* in: debug struct */ |
37 | #endif /* UNIV_SYNC_DEBUG */ | |
38 | ||
39 | +#ifdef HAVE_ATOMIC_BUILTINS | |
40 | +/* This value means NOT_LOCKED */ | |
41 | +#define RW_LOCK_BIAS 0x00100000 | |
337bc045 AM |
42 | +#else |
43 | +#error HAVE_ATOMIC_BUILTINS is not defined. Do you use enough new GCC or compatibles? | |
44 | +#error Or do you use exact options for CFLAGS? | |
45 | +#error e.g. (for x86_32): "-m32 -march=i586 -mtune=i686" | |
46 | +#error e.g. (for Sparc_64): "-m64 -mcpu=v9" | |
47 | +#error Otherwise, this build may be slower than normal version. | |
eccb488f ER |
48 | +#endif |
49 | + | |
50 | /* NOTE! The structure appears here only for the compiler to know its size. | |
51 | Do not use its fields directly! The structure used in the spin lock | |
52 | implementation of a read-write lock. Several threads may have a shared lock | |
45532174 | 53 | @@ -417,9 +438,9 @@ |
eccb488f ER |
54 | field. Then no new readers are allowed in. */ |
55 | ||
56 | struct rw_lock_struct { | |
57 | - os_event_t event; /* Used by sync0arr.c for thread queueing */ | |
58 | - | |
59 | -#ifdef __WIN__ | |
60 | + /* Used by sync0arr.c for thread queueing */ | |
61 | + os_event_t s_event; /* Used for s_lock */ | |
62 | + os_event_t x_event; /* Used for x_lock */ | |
63 | os_event_t wait_ex_event; /* This windows specific event is | |
64 | used by the thread which has set the | |
65 | lock state to RW_LOCK_WAIT_EX. The | |
45532174 | 66 | @@ -427,31 +448,35 @@ |
eccb488f ER |
67 | thread will be the next one to proceed |
68 | once the current the event gets | |
69 | signalled. See LEMMA 2 in sync0sync.c */ | |
70 | + | |
71 | +#ifdef HAVE_ATOMIC_BUILTINS | |
72 | + volatile lint lock_word; /* Used by using atomic builtin */ | |
73 | #endif | |
74 | ||
75 | - ulint reader_count; /* Number of readers who have locked this | |
76 | + volatile ulint reader_count; /* Number of readers who have locked this | |
77 | lock in the shared mode */ | |
78 | - ulint writer; /* This field is set to RW_LOCK_EX if there | |
79 | + volatile ulint writer; /* This field is set to RW_LOCK_EX if there | |
80 | is a writer owning the lock (in exclusive | |
81 | mode), RW_LOCK_WAIT_EX if a writer is | |
82 | queueing for the lock, and | |
83 | RW_LOCK_NOT_LOCKED, otherwise. */ | |
84 | - os_thread_id_t writer_thread; | |
85 | + volatile os_thread_id_t writer_thread; | |
86 | /* Thread id of a possible writer thread */ | |
87 | - ulint writer_count; /* Number of times the same thread has | |
88 | + volatile ulint writer_count; /* Number of times the same thread has | |
89 | recursively locked the lock in the exclusive | |
90 | mode */ | |
91 | +#ifndef HAVE_ATOMIC_BUILTINS | |
92 | mutex_t mutex; /* The mutex protecting rw_lock_struct */ | |
93 | +#endif | |
94 | ulint pass; /* Default value 0. This is set to some | |
95 | value != 0 given by the caller of an x-lock | |
96 | operation, if the x-lock is to be passed to | |
97 | another thread to unlock (which happens in | |
98 | asynchronous i/o). */ | |
99 | - ulint waiters; /* This ulint is set to 1 if there are | |
100 | - waiters (readers or writers) in the global | |
101 | - wait array, waiting for this rw_lock. | |
102 | - Otherwise, == 0. */ | |
103 | - ibool writer_is_wait_ex; | |
104 | + volatile ulint s_waiters; /* 1: there are waiters (s_lock) */ | |
105 | + volatile ulint x_waiters; /* 1: there are waiters (x_lock) */ | |
106 | + volatile ulint wait_ex_waiters; /* 1: there are waiters (wait_ex) */ | |
107 | + volatile ibool writer_is_wait_ex; | |
108 | /* This is TRUE if the writer field is | |
109 | RW_LOCK_WAIT_EX; this field is located far | |
110 | from the memory update hotspot fields which | |
45532174 ER |
111 | diff -ruN a/innobase/include/sync0rw.ic b/innobase/include/sync0rw.ic |
112 | --- a/innobase/include/sync0rw.ic 2009-01-30 06:42:20.000000000 +0900 | |
113 | +++ b/innobase/include/sync0rw.ic 2009-04-16 17:06:53.000000000 +0900 | |
114 | @@ -47,20 +47,64 @@ | |
eccb488f ER |
115 | Accessor functions for rw lock. */ |
116 | UNIV_INLINE | |
117 | ulint | |
118 | -rw_lock_get_waiters( | |
119 | +rw_lock_get_s_waiters( | |
120 | /*================*/ | |
121 | rw_lock_t* lock) | |
122 | { | |
123 | - return(lock->waiters); | |
124 | + return(lock->s_waiters); | |
45532174 ER |
125 | } |
126 | UNIV_INLINE | |
127 | -void | |
128 | -rw_lock_set_waiters( | |
eccb488f ER |
129 | +ulint |
130 | +rw_lock_get_x_waiters( | |
45532174 | 131 | /*================*/ |
eccb488f ER |
132 | + rw_lock_t* lock) |
133 | +{ | |
134 | + return(lock->x_waiters); | |
135 | +} | |
136 | +UNIV_INLINE | |
137 | +ulint | |
138 | +rw_lock_get_wx_waiters( | |
139 | +/*================*/ | |
140 | + rw_lock_t* lock) | |
141 | +{ | |
142 | + return(lock->wait_ex_waiters); | |
45532174 ER |
143 | +} |
144 | +UNIV_INLINE | |
145 | +void | |
eccb488f ER |
146 | +rw_lock_set_s_waiters( |
147 | rw_lock_t* lock, | |
148 | ulint flag) | |
149 | { | |
150 | - lock->waiters = flag; | |
45532174 ER |
151 | +#ifdef HAVE_ATOMIC_BUILTINS |
152 | + __sync_lock_test_and_set(&lock->s_waiters, flag); | |
153 | +#else | |
eccb488f | 154 | + lock->s_waiters = flag; |
45532174 | 155 | +#endif |
eccb488f ER |
156 | +} |
157 | +UNIV_INLINE | |
158 | +void | |
159 | +rw_lock_set_x_waiters( | |
160 | + rw_lock_t* lock, | |
161 | + ulint flag) | |
162 | +{ | |
45532174 ER |
163 | +#ifdef HAVE_ATOMIC_BUILTINS |
164 | + __sync_lock_test_and_set(&lock->x_waiters, flag); | |
165 | +#else | |
eccb488f | 166 | + lock->x_waiters = flag; |
45532174 | 167 | +#endif |
eccb488f ER |
168 | +} |
169 | +UNIV_INLINE | |
170 | +void | |
171 | +rw_lock_set_wx_waiters( | |
172 | +/*================*/ | |
173 | + rw_lock_t* lock, | |
174 | + ulint flag) | |
175 | +{ | |
45532174 ER |
176 | +#ifdef HAVE_ATOMIC_BUILTINS |
177 | + __sync_lock_test_and_set(&lock->wait_ex_waiters, flag); | |
178 | +#else | |
eccb488f | 179 | + lock->wait_ex_waiters = flag; |
45532174 | 180 | +#endif |
eccb488f ER |
181 | } |
182 | UNIV_INLINE | |
183 | ulint | |
45532174 | 184 | @@ -68,7 +112,19 @@ |
eccb488f ER |
185 | /*===============*/ |
186 | rw_lock_t* lock) | |
187 | { | |
188 | +#ifdef HAVE_ATOMIC_BUILTINS | |
189 | + if (lock->writer == RW_LOCK_NOT_LOCKED) { | |
190 | + return(RW_LOCK_NOT_LOCKED); | |
191 | + } | |
192 | + | |
193 | + if (lock->writer_is_wait_ex) { | |
194 | + return(RW_LOCK_WAIT_EX); | |
195 | + } else { | |
196 | + return(RW_LOCK_EX); | |
197 | + } | |
198 | +#else | |
199 | return(lock->writer); | |
200 | +#endif | |
201 | } | |
202 | UNIV_INLINE | |
203 | void | |
45532174 | 204 | @@ -96,6 +152,7 @@ |
eccb488f ER |
205 | { |
206 | lock->reader_count = count; | |
207 | } | |
208 | +#ifndef HAVE_ATOMIC_BUILTINS | |
209 | UNIV_INLINE | |
210 | mutex_t* | |
211 | rw_lock_get_mutex( | |
45532174 | 212 | @@ -104,6 +161,7 @@ |
eccb488f ER |
213 | { |
214 | return(&(lock->mutex)); | |
215 | } | |
216 | +#endif | |
217 | ||
218 | /********************************************************************** | |
219 | Returns the value of writer_count for the lock. Does not reserve the lock | |
45532174 | 220 | @@ -133,14 +191,26 @@ |
eccb488f ER |
221 | const char* file_name, /* in: file name where lock requested */ |
222 | ulint line) /* in: line where requested */ | |
223 | { | |
224 | -#ifdef UNIV_SYNC_DEBUG | |
225 | +#if defined(UNIV_SYNC_DEBUG) && !defined(HAVE_ATOMIC_BUILTINS) | |
226 | ut_ad(mutex_own(rw_lock_get_mutex(lock))); | |
227 | #endif /* UNIV_SYNC_DEBUG */ | |
228 | /* Check if the writer field is free */ | |
229 | ||
230 | +#ifdef HAVE_ATOMIC_BUILTINS | |
231 | + if (UNIV_LIKELY(rw_lock_get_writer(lock) == RW_LOCK_NOT_LOCKED)) { | |
232 | + /* try s-lock */ | |
233 | + if(__sync_sub_and_fetch(&(lock->lock_word),1) <= 0) { | |
234 | + /* fail */ | |
235 | + __sync_fetch_and_add(&(lock->lock_word),1); | |
236 | + return(FALSE); /* locking did not succeed */ | |
237 | + } | |
238 | + /* success */ | |
239 | + __sync_fetch_and_add(&(lock->reader_count),1); | |
240 | +#else | |
241 | if (UNIV_LIKELY(lock->writer == RW_LOCK_NOT_LOCKED)) { | |
242 | /* Set the shared lock by incrementing the reader count */ | |
243 | lock->reader_count++; | |
244 | +#endif | |
245 | ||
246 | #ifdef UNIV_SYNC_DEBUG | |
247 | rw_lock_add_debug_info(lock, pass, RW_LOCK_SHARED, file_name, | |
45532174 | 248 | @@ -167,11 +237,15 @@ |
eccb488f ER |
249 | const char* file_name, /* in: file name where requested */ |
250 | ulint line) /* in: line where lock requested */ | |
251 | { | |
252 | - ut_ad(lock->writer == RW_LOCK_NOT_LOCKED); | |
253 | + ut_ad(rw_lock_get_writer(lock) == RW_LOCK_NOT_LOCKED); | |
254 | ut_ad(rw_lock_get_reader_count(lock) == 0); | |
255 | ||
256 | /* Set the shared lock by incrementing the reader count */ | |
257 | +#ifdef HAVE_ATOMIC_BUILTINS | |
258 | + __sync_fetch_and_add(&(lock->reader_count),1); | |
259 | +#else | |
260 | lock->reader_count++; | |
261 | +#endif | |
262 | ||
263 | lock->last_s_file_name = file_name; | |
264 | lock->last_s_line = line; | |
45532174 | 265 | @@ -199,7 +273,11 @@ |
eccb488f ER |
266 | |
267 | rw_lock_set_writer(lock, RW_LOCK_EX); | |
268 | lock->writer_thread = os_thread_get_curr_id(); | |
269 | +#ifdef HAVE_ATOMIC_BUILTINS | |
270 | + __sync_fetch_and_add(&(lock->writer_count),1); | |
271 | +#else | |
272 | lock->writer_count++; | |
273 | +#endif | |
274 | lock->pass = 0; | |
275 | ||
276 | lock->last_x_file_name = file_name; | |
45532174 | 277 | @@ -241,15 +319,21 @@ |
eccb488f ER |
278 | ut_ad(!rw_lock_own(lock, RW_LOCK_SHARED)); /* see NOTE above */ |
279 | #endif /* UNIV_SYNC_DEBUG */ | |
280 | ||
281 | +#ifndef HAVE_ATOMIC_BUILTINS | |
282 | mutex_enter(rw_lock_get_mutex(lock)); | |
283 | +#endif | |
284 | ||
285 | if (UNIV_LIKELY(rw_lock_s_lock_low(lock, pass, file_name, line))) { | |
286 | +#ifndef HAVE_ATOMIC_BUILTINS | |
287 | mutex_exit(rw_lock_get_mutex(lock)); | |
288 | +#endif | |
289 | ||
290 | return; /* Success */ | |
291 | } else { | |
292 | /* Did not succeed, try spin wait */ | |
293 | +#ifndef HAVE_ATOMIC_BUILTINS | |
294 | mutex_exit(rw_lock_get_mutex(lock)); | |
295 | +#endif | |
296 | ||
297 | rw_lock_s_lock_spin(lock, pass, file_name, line); | |
298 | ||
45532174 | 299 | @@ -272,11 +356,23 @@ |
eccb488f ER |
300 | { |
301 | ibool success = FALSE; | |
302 | ||
303 | +#ifdef HAVE_ATOMIC_BUILTINS | |
304 | + if (rw_lock_get_writer(lock) == RW_LOCK_NOT_LOCKED) { | |
305 | + /* try s-lock */ | |
306 | + if(__sync_sub_and_fetch(&(lock->lock_word),1) <= 0) { | |
307 | + /* fail */ | |
308 | + __sync_fetch_and_add(&(lock->lock_word),1); | |
309 | + return(FALSE); /* locking did not succeed */ | |
310 | + } | |
311 | + /* success */ | |
312 | + __sync_fetch_and_add(&(lock->reader_count),1); | |
313 | +#else | |
314 | mutex_enter(rw_lock_get_mutex(lock)); | |
315 | ||
316 | if (lock->writer == RW_LOCK_NOT_LOCKED) { | |
317 | /* Set the shared lock by incrementing the reader count */ | |
318 | lock->reader_count++; | |
319 | +#endif | |
320 | ||
321 | #ifdef UNIV_SYNC_DEBUG | |
322 | rw_lock_add_debug_info(lock, 0, RW_LOCK_SHARED, file_name, | |
45532174 | 323 | @@ -289,7 +385,9 @@ |
eccb488f ER |
324 | success = TRUE; |
325 | } | |
326 | ||
327 | +#ifndef HAVE_ATOMIC_BUILTINS | |
328 | mutex_exit(rw_lock_get_mutex(lock)); | |
329 | +#endif | |
330 | ||
331 | return(success); | |
332 | } | |
45532174 | 333 | @@ -309,6 +407,54 @@ |
eccb488f ER |
334 | { |
335 | ibool success = FALSE; | |
336 | os_thread_id_t curr_thread = os_thread_get_curr_id(); | |
337 | +#ifdef HAVE_ATOMIC_BUILTINS | |
45532174 ER |
338 | + if (lock->reader_count == 0) { |
339 | + /* try to lock writer */ | |
340 | + if(__sync_lock_test_and_set(&(lock->writer),RW_LOCK_EX) | |
341 | + == RW_LOCK_NOT_LOCKED) { | |
eccb488f | 342 | + /* success */ |
45532174 ER |
343 | +retry_x_lock: |
344 | + /* try x-lock */ | |
345 | + if(__sync_sub_and_fetch(&(lock->lock_word), | |
346 | + RW_LOCK_BIAS) == 0) { | |
eccb488f ER |
347 | + /* success */ |
348 | + lock->writer_thread = curr_thread; | |
349 | + lock->pass = 0; | |
350 | + lock->writer_is_wait_ex = FALSE; | |
351 | + /* next function may work as memory barrier */ | |
352 | + relock: | |
353 | + __sync_fetch_and_add(&(lock->writer_count),1); | |
354 | + | |
355 | +#ifdef UNIV_SYNC_DEBUG | |
356 | + rw_lock_add_debug_info(lock, 0, RW_LOCK_EX, file_name, line); | |
357 | +#endif | |
358 | + | |
359 | + lock->last_x_file_name = file_name; | |
360 | + lock->last_x_line = line; | |
361 | + | |
362 | + ut_ad(rw_lock_validate(lock)); | |
363 | + | |
364 | + return(TRUE); | |
365 | + } else { | |
45532174 ER |
366 | + /* fail (x-lock) */ |
367 | + if (__sync_fetch_and_add(&(lock->lock_word),RW_LOCK_BIAS) | |
368 | + == 0) | |
369 | + goto retry_x_lock; | |
eccb488f | 370 | + } |
45532174 ER |
371 | + |
372 | + __sync_lock_test_and_set(&(lock->writer),RW_LOCK_NOT_LOCKED); | |
eccb488f ER |
373 | + } |
374 | + } | |
375 | + | |
376 | + if (lock->pass == 0 | |
45532174 | 377 | + && os_thread_eq(lock->writer_thread, curr_thread)) { |
eccb488f ER |
378 | + goto relock; |
379 | + } | |
380 | + | |
45532174 | 381 | + //ut_ad(rw_lock_validate(lock)); |
eccb488f ER |
382 | + |
383 | + return(FALSE); | |
384 | +#else | |
385 | mutex_enter(rw_lock_get_mutex(lock)); | |
386 | ||
387 | if (UNIV_UNLIKELY(rw_lock_get_reader_count(lock) != 0)) { | |
45532174 | 388 | @@ -339,6 +485,7 @@ |
eccb488f ER |
389 | ut_ad(rw_lock_validate(lock)); |
390 | ||
391 | return(success); | |
392 | +#endif | |
393 | } | |
394 | ||
395 | /********************************************************************** | |
45532174 | 396 | @@ -354,16 +501,33 @@ |
eccb488f ER |
397 | #endif |
398 | ) | |
399 | { | |
400 | +#ifndef HAVE_ATOMIC_BUILTINS | |
401 | mutex_t* mutex = &(lock->mutex); | |
402 | - ibool sg = FALSE; | |
403 | +#endif | |
404 | + ibool x_sg = FALSE; | |
405 | + ibool wx_sg = FALSE; | |
406 | +#ifdef HAVE_ATOMIC_BUILTINS | |
407 | + ibool last = FALSE; | |
408 | +#endif | |
409 | ||
410 | +#ifndef HAVE_ATOMIC_BUILTINS | |
411 | /* Acquire the mutex protecting the rw-lock fields */ | |
412 | mutex_enter(mutex); | |
413 | +#endif | |
414 | ||
415 | /* Reset the shared lock by decrementing the reader count */ | |
416 | ||
417 | ut_a(lock->reader_count > 0); | |
418 | +#ifdef HAVE_ATOMIC_BUILTINS | |
419 | + /* unlock lock_word */ | |
420 | + __sync_fetch_and_add(&(lock->lock_word),1); | |
421 | + | |
422 | + if(__sync_sub_and_fetch(&(lock->reader_count),1) == 0) { | |
423 | + last = TRUE; | |
424 | + } | |
425 | +#else | |
426 | lock->reader_count--; | |
427 | +#endif | |
428 | ||
429 | #ifdef UNIV_SYNC_DEBUG | |
430 | rw_lock_remove_debug_info(lock, pass, RW_LOCK_SHARED); | |
45532174 | 431 | @@ -372,22 +536,39 @@ |
eccb488f ER |
432 | /* If there may be waiters and this was the last s-lock, |
433 | signal the object */ | |
434 | ||
435 | - if (UNIV_UNLIKELY(lock->waiters) | |
436 | +#ifdef HAVE_ATOMIC_BUILTINS | |
45532174 ER |
437 | + if (UNIV_UNLIKELY(last && __sync_lock_test_and_set(&lock->wait_ex_waiters, 0))) { |
438 | + os_event_set(lock->wait_ex_event); | |
439 | + sync_array_object_signalled(sync_primary_wait_array); | |
440 | + } | |
441 | + else if (UNIV_UNLIKELY(last && __sync_lock_test_and_set(&lock->x_waiters, 0))) { | |
442 | + os_event_set(lock->x_event); | |
443 | + sync_array_object_signalled(sync_primary_wait_array); | |
444 | + } | |
eccb488f ER |
445 | +#else |
446 | + if (UNIV_UNLIKELY(lock->wait_ex_waiters) | |
447 | && lock->reader_count == 0) { | |
448 | - sg = TRUE; | |
eccb488f ER |
449 | + wx_sg = TRUE; |
450 | ||
451 | - rw_lock_set_waiters(lock, 0); | |
452 | + rw_lock_set_wx_waiters(lock, 0); | |
453 | + } | |
eccb488f ER |
454 | + else if (UNIV_UNLIKELY(lock->x_waiters) |
455 | + && lock->reader_count == 0) { | |
eccb488f ER |
456 | + x_sg = TRUE; |
457 | + | |
458 | + rw_lock_set_x_waiters(lock, 0); | |
459 | } | |
460 | ||
eccb488f | 461 | mutex_exit(mutex); |
eccb488f ER |
462 | |
463 | - if (UNIV_UNLIKELY(sg)) { | |
464 | -#ifdef __WIN__ | |
465 | + if (UNIV_UNLIKELY(wx_sg)) { | |
466 | os_event_set(lock->wait_ex_event); | |
467 | -#endif | |
468 | - os_event_set(lock->event); | |
469 | + sync_array_object_signalled(sync_primary_wait_array); | |
470 | + } else if (UNIV_UNLIKELY(x_sg)) { | |
471 | + os_event_set(lock->x_event); | |
472 | sync_array_object_signalled(sync_primary_wait_array); | |
473 | } | |
45532174 ER |
474 | +#endif |
475 | ||
476 | ut_ad(rw_lock_validate(lock)); | |
eccb488f | 477 | |
45532174 | 478 | @@ -409,13 +590,22 @@ |
eccb488f ER |
479 | |
480 | ut_ad(lock->reader_count > 0); | |
481 | ||
482 | +#ifdef HAVE_ATOMIC_BUILTINS | |
483 | + __sync_sub_and_fetch(&(lock->reader_count),1); | |
484 | +#else | |
485 | lock->reader_count--; | |
486 | +#endif | |
487 | ||
488 | #ifdef UNIV_SYNC_DEBUG | |
489 | rw_lock_remove_debug_info(lock, 0, RW_LOCK_SHARED); | |
490 | #endif | |
491 | ||
492 | +#ifdef HAVE_ATOMIC_BUILTINS | |
493 | + ut_ad(!lock->s_waiters); | |
494 | + ut_ad(!lock->x_waiters); | |
495 | +#else | |
496 | ut_ad(!lock->waiters); | |
497 | +#endif | |
498 | ut_ad(rw_lock_validate(lock)); | |
499 | #ifdef UNIV_SYNC_PERF_STAT | |
500 | rw_s_exit_count++; | |
45532174 | 501 | @@ -435,41 +625,83 @@ |
eccb488f ER |
502 | #endif |
503 | ) | |
504 | { | |
505 | - ibool sg = FALSE; | |
506 | +#ifdef HAVE_ATOMIC_BUILTINS | |
507 | + ibool last = FALSE; | |
508 | +#endif | |
509 | + ibool s_sg = FALSE; | |
510 | + ibool x_sg = FALSE; | |
511 | ||
512 | +#ifndef HAVE_ATOMIC_BUILTINS | |
513 | /* Acquire the mutex protecting the rw-lock fields */ | |
514 | mutex_enter(&(lock->mutex)); | |
515 | +#endif | |
516 | ||
517 | /* Reset the exclusive lock if this thread no longer has an x-mode | |
518 | lock */ | |
519 | ||
520 | ut_ad(lock->writer_count > 0); | |
521 | ||
522 | +#ifdef HAVE_ATOMIC_BUILTINS | |
523 | + if(__sync_sub_and_fetch(&(lock->writer_count),1) == 0) { | |
524 | + last = TRUE; | |
525 | + } | |
526 | + | |
527 | + if (last) { | |
528 | + /* unlock lock_word */ | |
529 | + __sync_fetch_and_add(&(lock->lock_word),RW_LOCK_BIAS); | |
530 | + | |
531 | + /* FIXME: It is a value of bad manners for pthread. | |
532 | + But we shouldn't keep an ID of not-owner. */ | |
533 | + lock->writer_thread = -1; | |
45532174 | 534 | + __sync_lock_test_and_set(&(lock->writer),RW_LOCK_NOT_LOCKED); |
eccb488f ER |
535 | + } |
536 | +#else | |
537 | lock->writer_count--; | |
538 | ||
539 | if (lock->writer_count == 0) { | |
540 | rw_lock_set_writer(lock, RW_LOCK_NOT_LOCKED); | |
541 | } | |
542 | +#endif | |
543 | ||
544 | #ifdef UNIV_SYNC_DEBUG | |
545 | rw_lock_remove_debug_info(lock, pass, RW_LOCK_EX); | |
546 | #endif | |
547 | ||
548 | /* If there may be waiters, signal the lock */ | |
549 | - if (UNIV_UNLIKELY(lock->waiters) | |
550 | - && lock->writer_count == 0) { | |
551 | - | |
552 | - sg = TRUE; | |
553 | - rw_lock_set_waiters(lock, 0); | |
554 | +#ifdef HAVE_ATOMIC_BUILTINS | |
555 | + if (last) { | |
45532174 ER |
556 | + if(__sync_lock_test_and_set(&lock->s_waiters, 0)){ |
557 | + s_sg = TRUE; | |
558 | + } | |
559 | + if(__sync_lock_test_and_set(&lock->x_waiters, 0)){ | |
560 | + x_sg = TRUE; | |
561 | + } | |
562 | + } | |
eccb488f ER |
563 | +#else |
564 | + if (lock->writer_count == 0) { | |
eccb488f ER |
565 | + if(lock->s_waiters){ |
566 | + s_sg = TRUE; | |
567 | + rw_lock_set_s_waiters(lock, 0); | |
568 | + } | |
569 | + if(lock->x_waiters){ | |
570 | + x_sg = TRUE; | |
571 | + rw_lock_set_x_waiters(lock, 0); | |
572 | + } | |
573 | } | |
574 | ||
eccb488f ER |
575 | mutex_exit(&(lock->mutex)); |
576 | +#endif | |
577 | ||
578 | - if (UNIV_UNLIKELY(sg)) { | |
579 | + if (UNIV_UNLIKELY(s_sg)) { | |
580 | + os_event_set(lock->s_event); | |
581 | + sync_array_object_signalled(sync_primary_wait_array); | |
582 | + } | |
583 | + if (UNIV_UNLIKELY(x_sg)) { | |
584 | #ifdef __WIN__ | |
585 | + /* I doubt the necessity of it. */ | |
586 | os_event_set(lock->wait_ex_event); | |
587 | #endif | |
588 | - os_event_set(lock->event); | |
589 | + os_event_set(lock->x_event); | |
590 | sync_array_object_signalled(sync_primary_wait_array); | |
591 | } | |
592 | ||
45532174 | 593 | @@ -494,9 +726,13 @@ |
eccb488f ER |
594 | |
595 | ut_ad(lock->writer_count > 0); | |
596 | ||
597 | +#ifdef HAVE_ATOMIC_BUILTINS | |
598 | + if(__sync_sub_and_fetch(&(lock->writer_count),1) == 0) { | |
599 | +#else | |
600 | lock->writer_count--; | |
601 | ||
602 | if (lock->writer_count == 0) { | |
603 | +#endif | |
604 | rw_lock_set_writer(lock, RW_LOCK_NOT_LOCKED); | |
605 | } | |
606 | ||
45532174 | 607 | @@ -504,7 +740,12 @@ |
eccb488f ER |
608 | rw_lock_remove_debug_info(lock, 0, RW_LOCK_EX); |
609 | #endif | |
610 | ||
611 | +#ifdef HAVE_ATOMIC_BUILTINS | |
612 | + ut_ad(!lock->s_waiters); | |
613 | + ut_ad(!lock->x_waiters); | |
614 | +#else | |
615 | ut_ad(!lock->waiters); | |
616 | +#endif | |
617 | ut_ad(rw_lock_validate(lock)); | |
618 | ||
619 | #ifdef UNIV_SYNC_PERF_STAT | |
45532174 ER |
620 | diff -ruN a/innobase/sync/sync0arr.c b/innobase/sync/sync0arr.c |
621 | --- a/innobase/sync/sync0arr.c 2009-01-30 06:42:24.000000000 +0900 | |
622 | +++ b/innobase/sync/sync0arr.c 2009-04-16 16:15:28.000000000 +0900 | |
eccb488f ER |
623 | @@ -309,13 +309,13 @@ |
624 | { | |
625 | if (type == SYNC_MUTEX) { | |
626 | return(os_event_reset(((mutex_t *) object)->event)); | |
627 | -#ifdef __WIN__ | |
628 | } else if (type == RW_LOCK_WAIT_EX) { | |
629 | return(os_event_reset( | |
630 | ((rw_lock_t *) object)->wait_ex_event)); | |
631 | -#endif | |
632 | - } else { | |
633 | - return(os_event_reset(((rw_lock_t *) object)->event)); | |
634 | + } else if (type == RW_LOCK_SHARED) { | |
635 | + return(os_event_reset(((rw_lock_t *) object)->s_event)); | |
636 | + } else { /* RW_LOCK_EX */ | |
637 | + return(os_event_reset(((rw_lock_t *) object)->x_event)); | |
638 | } | |
639 | } | |
640 | ||
641 | @@ -415,15 +415,12 @@ | |
642 | ||
643 | if (cell->request_type == SYNC_MUTEX) { | |
644 | event = ((mutex_t*) cell->wait_object)->event; | |
645 | -#ifdef __WIN__ | |
646 | - /* On windows if the thread about to wait is the one which | |
647 | - has set the state of the rw_lock to RW_LOCK_WAIT_EX, then | |
648 | - it waits on a special event i.e.: wait_ex_event. */ | |
649 | } else if (cell->request_type == RW_LOCK_WAIT_EX) { | |
650 | event = ((rw_lock_t*) cell->wait_object)->wait_ex_event; | |
651 | -#endif | |
652 | - } else { | |
653 | - event = ((rw_lock_t*) cell->wait_object)->event; | |
654 | + } else if (cell->request_type == RW_LOCK_SHARED) { | |
655 | + event = ((rw_lock_t*) cell->wait_object)->s_event; | |
656 | + } else { | |
657 | + event = ((rw_lock_t*) cell->wait_object)->x_event; | |
658 | } | |
659 | ||
660 | cell->waiting = TRUE; | |
661 | @@ -464,6 +461,7 @@ | |
662 | mutex_t* mutex; | |
663 | rw_lock_t* rwlock; | |
664 | ulint type; | |
665 | + ulint writer; | |
666 | ||
667 | type = cell->request_type; | |
668 | ||
669 | @@ -492,12 +490,10 @@ | |
670 | (ulong) mutex->waiters); | |
671 | ||
672 | } else if (type == RW_LOCK_EX | |
673 | -#ifdef __WIN__ | |
674 | || type == RW_LOCK_WAIT_EX | |
675 | -#endif | |
676 | || type == RW_LOCK_SHARED) { | |
677 | ||
678 | - fputs(type == RW_LOCK_EX ? "X-lock on" : "S-lock on", file); | |
679 | + fputs(type == RW_LOCK_SHARED ? "S-lock on" : "X-lock on", file); | |
680 | ||
681 | rwlock = cell->old_wait_rw_lock; | |
682 | ||
683 | @@ -505,21 +501,23 @@ | |
684 | " RW-latch at %p created in file %s line %lu\n", | |
685 | rwlock, rwlock->cfile_name, | |
686 | (ulong) rwlock->cline); | |
687 | - if (rwlock->writer != RW_LOCK_NOT_LOCKED) { | |
688 | + writer = rw_lock_get_writer(rwlock); | |
689 | + if (writer != RW_LOCK_NOT_LOCKED) { | |
690 | fprintf(file, | |
691 | "a writer (thread id %lu) has reserved it in mode %s", | |
692 | (ulong) os_thread_pf(rwlock->writer_thread), | |
693 | - rwlock->writer == RW_LOCK_EX | |
694 | + writer == RW_LOCK_EX | |
695 | ? " exclusive\n" | |
696 | : " wait exclusive\n"); | |
697 | } | |
698 | ||
699 | fprintf(file, | |
700 | - "number of readers %lu, waiters flag %lu\n" | |
701 | + "number of readers %lu, s_waiters flag %lu, x_waiters flag %lu\n" | |
702 | "Last time read locked in file %s line %lu\n" | |
703 | "Last time write locked in file %s line %lu\n", | |
704 | (ulong) rwlock->reader_count, | |
705 | - (ulong) rwlock->waiters, | |
706 | + (ulong) rwlock->s_waiters, | |
707 | + (ulong) (rwlock->x_waiters || rwlock->wait_ex_waiters), | |
708 | rwlock->last_s_file_name, | |
709 | (ulong) rwlock->last_s_line, | |
710 | rwlock->last_x_file_name, | |
711 | @@ -839,11 +837,15 @@ | |
712 | /*========================*/ | |
713 | sync_array_t* arr) /* in: wait array */ | |
714 | { | |
715 | +#ifdef HAVE_ATOMIC_BUILTINS | |
716 | + __sync_fetch_and_add(&(arr->sg_count),1); | |
717 | +#else | |
718 | sync_array_enter(arr); | |
719 | ||
720 | arr->sg_count++; | |
721 | ||
722 | sync_array_exit(arr); | |
723 | +#endif | |
724 | } | |
725 | ||
726 | /************************************************************************** | |
727 | @@ -880,19 +882,23 @@ | |
728 | ||
729 | mutex = cell->wait_object; | |
730 | os_event_set(mutex->event); | |
731 | -#ifdef __WIN__ | |
732 | } else if (cell->request_type | |
733 | == RW_LOCK_WAIT_EX) { | |
734 | rw_lock_t* lock; | |
735 | ||
736 | lock = cell->wait_object; | |
737 | os_event_set(lock->wait_ex_event); | |
738 | -#endif | |
739 | - } else { | |
740 | + } else if (cell->request_type | |
741 | + == RW_LOCK_SHARED) { | |
742 | rw_lock_t* lock; | |
743 | ||
744 | lock = cell->wait_object; | |
745 | - os_event_set(lock->event); | |
746 | + os_event_set(lock->s_event); | |
747 | + } else { | |
748 | + rw_lock_t* lock; | |
749 | + | |
750 | + lock = cell->wait_object; | |
751 | + os_event_set(lock->x_event); | |
752 | } | |
753 | } | |
754 | } | |
45532174 ER |
755 | diff -ruN a/innobase/sync/sync0rw.c b/innobase/sync/sync0rw.c |
756 | --- a/innobase/sync/sync0rw.c 2009-01-30 06:42:24.000000000 +0900 | |
757 | +++ b/innobase/sync/sync0rw.c 2009-04-16 17:33:59.000000000 +0900 | |
eccb488f ER |
758 | @@ -99,6 +99,7 @@ |
759 | object is created, then the following call initializes | |
760 | the sync system. */ | |
761 | ||
762 | +#ifndef HAVE_ATOMIC_BUILTINS | |
763 | mutex_create(rw_lock_get_mutex(lock)); | |
764 | mutex_set_level(rw_lock_get_mutex(lock), SYNC_NO_ORDER_CHECK); | |
765 | ||
766 | @@ -108,8 +109,14 @@ | |
767 | lock->mutex.cmutex_name = cmutex_name; | |
768 | lock->mutex.mutex_type = 1; | |
769 | #endif /* UNIV_DEBUG && !UNIV_HOTBACKUP */ | |
770 | +#endif /* !HAVE_ATOMIC_BUILTINS */ | |
771 | ||
772 | - rw_lock_set_waiters(lock, 0); | |
773 | +#ifdef HAVE_ATOMIC_BUILTINS | |
774 | + lock->lock_word = RW_LOCK_BIAS; | |
775 | +#endif | |
776 | + rw_lock_set_s_waiters(lock, 0); | |
777 | + rw_lock_set_x_waiters(lock, 0); | |
778 | + rw_lock_set_wx_waiters(lock, 0); | |
779 | rw_lock_set_writer(lock, RW_LOCK_NOT_LOCKED); | |
780 | lock->writer_count = 0; | |
781 | rw_lock_set_reader_count(lock, 0); | |
782 | @@ -130,11 +137,9 @@ | |
783 | lock->last_x_file_name = "not yet reserved"; | |
784 | lock->last_s_line = 0; | |
785 | lock->last_x_line = 0; | |
786 | - lock->event = os_event_create(NULL); | |
787 | - | |
788 | -#ifdef __WIN__ | |
789 | + lock->s_event = os_event_create(NULL); | |
790 | + lock->x_event = os_event_create(NULL); | |
791 | lock->wait_ex_event = os_event_create(NULL); | |
792 | -#endif | |
793 | ||
794 | mutex_enter(&rw_lock_list_mutex); | |
795 | ||
796 | @@ -162,19 +167,21 @@ | |
797 | ut_a(rw_lock_validate(lock)); | |
798 | #endif /* UNIV_DEBUG */ | |
799 | ut_a(rw_lock_get_writer(lock) == RW_LOCK_NOT_LOCKED); | |
800 | - ut_a(rw_lock_get_waiters(lock) == 0); | |
801 | + ut_a(rw_lock_get_s_waiters(lock) == 0); | |
802 | + ut_a(rw_lock_get_x_waiters(lock) == 0); | |
803 | + ut_a(rw_lock_get_wx_waiters(lock) == 0); | |
804 | ut_a(rw_lock_get_reader_count(lock) == 0); | |
805 | ||
806 | lock->magic_n = 0; | |
807 | ||
808 | +#ifndef HAVE_ATOMIC_BUILTINS | |
809 | mutex_free(rw_lock_get_mutex(lock)); | |
810 | +#endif | |
811 | ||
812 | mutex_enter(&rw_lock_list_mutex); | |
813 | - os_event_free(lock->event); | |
814 | - | |
815 | -#ifdef __WIN__ | |
816 | + os_event_free(lock->s_event); | |
817 | + os_event_free(lock->x_event); | |
818 | os_event_free(lock->wait_ex_event); | |
819 | -#endif | |
820 | ||
821 | if (UT_LIST_GET_PREV(list, lock)) { | |
822 | ut_a(UT_LIST_GET_PREV(list, lock)->magic_n == RW_LOCK_MAGIC_N); | |
45532174 | 823 | @@ -192,26 +199,43 @@ |
eccb488f ER |
824 | Checks that the rw-lock has been initialized and that there are no |
825 | simultaneous shared and exclusive locks. */ | |
826 | ||
827 | +/* MEMO: If HAVE_ATOMIC_BUILTINS, we should use this function statically. */ | |
828 | + | |
829 | ibool | |
830 | rw_lock_validate( | |
831 | /*=============*/ | |
45532174 | 832 | rw_lock_t* lock) |
eccb488f | 833 | { |
45532174 | 834 | + ulint test; |
eccb488f ER |
835 | ut_a(lock); |
836 | ||
837 | +#ifndef HAVE_ATOMIC_BUILTINS | |
838 | mutex_enter(rw_lock_get_mutex(lock)); | |
839 | +#endif | |
840 | ||
841 | ut_a(lock->magic_n == RW_LOCK_MAGIC_N); | |
45532174 | 842 | +#ifndef HAVE_ATOMIC_BUILTINS |
eccb488f | 843 | ut_a((rw_lock_get_reader_count(lock) == 0) |
45532174 ER |
844 | || (rw_lock_get_writer(lock) != RW_LOCK_EX)); |
845 | - ut_a((rw_lock_get_writer(lock) == RW_LOCK_EX) | |
846 | - || (rw_lock_get_writer(lock) == RW_LOCK_WAIT_EX) | |
847 | - || (rw_lock_get_writer(lock) == RW_LOCK_NOT_LOCKED)); | |
eccb488f ER |
848 | - ut_a((rw_lock_get_waiters(lock) == 0) |
849 | - || (rw_lock_get_waiters(lock) == 1)); | |
45532174 ER |
850 | +#endif |
851 | + test = rw_lock_get_writer(lock); | |
852 | + ut_a((test == RW_LOCK_EX) | |
853 | + || (test == RW_LOCK_WAIT_EX) | |
854 | + || (test == RW_LOCK_NOT_LOCKED)); | |
855 | + test = rw_lock_get_s_waiters(lock); | |
856 | + ut_a((test == 0) | |
857 | + || (test == 1)); | |
858 | + test = rw_lock_get_x_waiters(lock); | |
859 | + ut_a((test == 0) | |
860 | + || (test == 1)); | |
861 | + test = rw_lock_get_wx_waiters(lock); | |
862 | + ut_a((test == 0) | |
863 | + || (test == 1)); | |
864 | +#ifndef HAVE_ATOMIC_BUILTINS | |
eccb488f ER |
865 | ut_a((lock->writer != RW_LOCK_EX) || (lock->writer_count > 0)); |
866 | ||
eccb488f ER |
867 | mutex_exit(rw_lock_get_mutex(lock)); |
868 | +#endif | |
869 | ||
870 | return(TRUE); | |
871 | } | |
45532174 | 872 | @@ -237,13 +261,14 @@ |
eccb488f ER |
873 | ut_ad(rw_lock_validate(lock)); |
874 | ||
875 | lock_loop: | |
876 | + i = 0; | |
877 | +spin_loop: | |
878 | rw_s_spin_wait_count++; | |
879 | ||
880 | /* Spin waiting for the writer field to become free */ | |
881 | - i = 0; | |
882 | ||
883 | - while (rw_lock_get_writer(lock) != RW_LOCK_NOT_LOCKED | |
884 | - && i < SYNC_SPIN_ROUNDS) { | |
885 | + while (i < SYNC_SPIN_ROUNDS | |
886 | + && rw_lock_get_writer(lock) != RW_LOCK_NOT_LOCKED) { | |
887 | if (srv_spin_wait_delay) { | |
888 | ut_delay(ut_rnd_interval(0, srv_spin_wait_delay)); | |
889 | } | |
45532174 | 890 | @@ -262,15 +287,27 @@ |
eccb488f ER |
891 | lock->cfile_name, (ulong) lock->cline, (ulong) i); |
892 | } | |
893 | ||
894 | +#ifndef HAVE_ATOMIC_BUILTINS | |
895 | mutex_enter(rw_lock_get_mutex(lock)); | |
896 | +#endif | |
897 | ||
898 | /* We try once again to obtain the lock */ | |
899 | ||
900 | if (TRUE == rw_lock_s_lock_low(lock, pass, file_name, line)) { | |
901 | +#ifndef HAVE_ATOMIC_BUILTINS | |
902 | mutex_exit(rw_lock_get_mutex(lock)); | |
903 | +#endif | |
904 | ||
905 | return; /* Success */ | |
906 | } else { | |
907 | +#ifdef HAVE_ATOMIC_BUILTINS | |
908 | + /* like sync0sync.c doing */ | |
909 | + i++; | |
910 | + | |
911 | + if (i < SYNC_SPIN_ROUNDS) { | |
912 | + goto spin_loop; | |
913 | + } | |
914 | +#endif | |
915 | /* If we get here, locking did not succeed, we may | |
916 | suspend the thread to wait in the wait array */ | |
917 | ||
45532174 | 918 | @@ -281,9 +318,26 @@ |
eccb488f ER |
919 | file_name, line, |
920 | &index); | |
921 | ||
922 | - rw_lock_set_waiters(lock, 1); | |
923 | + rw_lock_set_s_waiters(lock, 1); | |
45532174 | 924 | + |
eccb488f ER |
925 | +#ifdef HAVE_ATOMIC_BUILTINS |
926 | + /* like sync0sync.c doing */ | |
927 | + for (i = 0; i < 4; i++) { | |
928 | + if (TRUE == rw_lock_s_lock_low(lock, pass, file_name, line)) { | |
929 | + sync_array_free_cell(sync_primary_wait_array, index); | |
930 | + return; /* Success */ | |
931 | + } | |
932 | + } | |
45532174 | 933 | |
337bc045 | 934 | + /* If wait_ex_waiter stalls, wakes it. */ |
45532174 ER |
935 | + if (lock->reader_count == 0 |
936 | + && __sync_lock_test_and_set(&lock->wait_ex_waiters, 0)) { | |
337bc045 AM |
937 | + os_event_set(lock->wait_ex_event); |
938 | + sync_array_object_signalled(sync_primary_wait_array); | |
939 | + } | |
eccb488f ER |
940 | +#else |
941 | mutex_exit(rw_lock_get_mutex(lock)); | |
942 | +#endif | |
943 | ||
944 | if (srv_print_latch_waits) { | |
945 | fprintf(stderr, | |
45532174 | 946 | @@ -318,13 +372,19 @@ |
eccb488f ER |
947 | { |
948 | ut_ad(rw_lock_is_locked(lock, RW_LOCK_EX)); | |
949 | ||
950 | +#ifndef HAVE_ATOMIC_BUILTINS | |
951 | mutex_enter(&(lock->mutex)); | |
952 | +#endif | |
953 | ||
954 | lock->writer_thread = os_thread_get_curr_id(); | |
955 | ||
956 | lock->pass = 0; | |
957 | ||
958 | +#ifndef HAVE_ATOMIC_BUILTINS | |
959 | mutex_exit(&(lock->mutex)); | |
960 | +#else | |
961 | + __sync_synchronize(); | |
962 | +#endif | |
963 | } | |
964 | ||
965 | /********************************************************************** | |
45532174 | 966 | @@ -342,6 +402,89 @@ |
eccb488f ER |
967 | const char* file_name,/* in: file name where lock requested */ |
968 | ulint line) /* in: line where requested */ | |
969 | { | |
970 | +#ifdef HAVE_ATOMIC_BUILTINS | |
971 | + os_thread_id_t curr_thread = os_thread_get_curr_id(); | |
45532174 | 972 | +retry_writer: |
eccb488f ER |
973 | + /* try to lock writer */ |
974 | + if(__sync_lock_test_and_set(&(lock->writer),RW_LOCK_EX) | |
975 | + == RW_LOCK_NOT_LOCKED) { | |
976 | + /* success */ | |
977 | + /* obtain RW_LOCK_WAIT_EX right */ | |
978 | + lock->writer_thread = curr_thread; | |
979 | + lock->pass = pass; | |
980 | + lock->writer_is_wait_ex = TRUE; | |
981 | + /* atomic operation may be safer about memory order. */ | |
982 | + __sync_synchronize(); | |
983 | +#ifdef UNIV_SYNC_DEBUG | |
984 | + rw_lock_add_debug_info(lock, pass, RW_LOCK_WAIT_EX, | |
985 | + file_name, line); | |
986 | +#endif | |
987 | + } | |
988 | + | |
989 | + if (!os_thread_eq(lock->writer_thread, curr_thread)) { | |
990 | + return(RW_LOCK_NOT_LOCKED); | |
991 | + } | |
992 | + | |
993 | + switch(rw_lock_get_writer(lock)) { | |
994 | + case RW_LOCK_WAIT_EX: | |
995 | + /* have right to try x-lock */ | |
337bc045 AM |
996 | +retry_x_lock: |
997 | + /* try x-lock */ | |
998 | + if(__sync_sub_and_fetch(&(lock->lock_word), | |
999 | + RW_LOCK_BIAS) == 0) { | |
1000 | + /* success */ | |
1001 | + lock->pass = pass; | |
1002 | + lock->writer_is_wait_ex = FALSE; | |
1003 | + __sync_fetch_and_add(&(lock->writer_count),1); | |
eccb488f ER |
1004 | + |
1005 | +#ifdef UNIV_SYNC_DEBUG | |
337bc045 AM |
1006 | + rw_lock_remove_debug_info(lock, pass, RW_LOCK_WAIT_EX); |
1007 | + rw_lock_add_debug_info(lock, pass, RW_LOCK_EX, | |
1008 | + file_name, line); | |
eccb488f ER |
1009 | +#endif |
1010 | + | |
337bc045 AM |
1011 | + lock->last_x_file_name = file_name; |
1012 | + lock->last_x_line = line; | |
eccb488f | 1013 | + |
337bc045 AM |
1014 | + /* Locking succeeded, we may return */ |
1015 | + return(RW_LOCK_EX); | |
1016 | + } else if(__sync_fetch_and_add(&(lock->lock_word), | |
1017 | + RW_LOCK_BIAS) == 0) { | |
1018 | + /* retry x-lock */ | |
1019 | + goto retry_x_lock; | |
eccb488f | 1020 | + } |
337bc045 | 1021 | + |
eccb488f ER |
1022 | + /* There are readers, we have to wait */ |
1023 | + return(RW_LOCK_WAIT_EX); | |
1024 | + | |
1025 | + break; | |
1026 | + | |
1027 | + case RW_LOCK_EX: | |
1028 | + /* already have x-lock */ | |
1029 | + if ((lock->pass == 0)&&(pass == 0)) { | |
1030 | + __sync_fetch_and_add(&(lock->writer_count),1); | |
1031 | + | |
1032 | +#ifdef UNIV_SYNC_DEBUG | |
1033 | + rw_lock_add_debug_info(lock, pass, RW_LOCK_EX, file_name, | |
1034 | + line); | |
1035 | +#endif | |
1036 | + | |
1037 | + lock->last_x_file_name = file_name; | |
1038 | + lock->last_x_line = line; | |
1039 | + | |
1040 | + /* Locking succeeded, we may return */ | |
1041 | + return(RW_LOCK_EX); | |
1042 | + } | |
1043 | + | |
1044 | + return(RW_LOCK_NOT_LOCKED); | |
1045 | + | |
1046 | + break; | |
1047 | + | |
45532174 ER |
1048 | + default: /* RW_LOCK_NOT_LOCKED? maybe impossible */ |
1049 | + goto retry_writer; | |
eccb488f ER |
1050 | + } |
1051 | +#else /* HAVE_ATOMIC_BUILTINS */ | |
1052 | + | |
1053 | #ifdef UNIV_SYNC_DEBUG | |
1054 | ut_ad(mutex_own(rw_lock_get_mutex(lock))); | |
1055 | #endif /* UNIV_SYNC_DEBUG */ | |
45532174 | 1056 | @@ -423,6 +566,7 @@ |
eccb488f ER |
1057 | /* Locking succeeded, we may return */ |
1058 | return(RW_LOCK_EX); | |
1059 | } | |
1060 | +#endif /* HAVE_ATOMIC_BUILTINS */ | |
1061 | ||
1062 | /* Locking did not succeed */ | |
1063 | return(RW_LOCK_NOT_LOCKED); | |
45532174 | 1064 | @@ -448,19 +592,33 @@ |
eccb488f ER |
1065 | ulint line) /* in: line where requested */ |
1066 | { | |
1067 | ulint index; /* index of the reserved wait cell */ | |
1068 | - ulint state; /* lock state acquired */ | |
1069 | + ulint state = RW_LOCK_NOT_LOCKED; /* lock state acquired */ | |
1070 | +#ifdef HAVE_ATOMIC_BUILTINS | |
1071 | + ulint prev_state = RW_LOCK_NOT_LOCKED; | |
1072 | +#endif | |
1073 | ulint i; /* spin round count */ | |
1074 | ||
1075 | ut_ad(rw_lock_validate(lock)); | |
1076 | ||
1077 | lock_loop: | |
1078 | + i = 0; | |
1079 | + | |
1080 | +#ifdef HAVE_ATOMIC_BUILTINS | |
1081 | + prev_state = state; | |
1082 | +#else | |
1083 | /* Acquire the mutex protecting the rw-lock fields */ | |
1084 | mutex_enter_fast(&(lock->mutex)); | |
1085 | +#endif | |
1086 | ||
1087 | state = rw_lock_x_lock_low(lock, pass, file_name, line); | |
1088 | ||
1089 | +#ifdef HAVE_ATOMIC_BUILTINS | |
1090 | + if (state != prev_state) i=0; /* if progress, reset counter. */ | |
1091 | +#else | |
1092 | mutex_exit(&(lock->mutex)); | |
1093 | +#endif | |
1094 | ||
1095 | +spin_loop: | |
1096 | if (state == RW_LOCK_EX) { | |
1097 | ||
1098 | return; /* Locking succeeded */ | |
45532174 | 1099 | @@ -468,10 +626,9 @@ |
eccb488f ER |
1100 | } else if (state == RW_LOCK_NOT_LOCKED) { |
1101 | ||
1102 | /* Spin waiting for the writer field to become free */ | |
1103 | - i = 0; | |
1104 | ||
1105 | - while (rw_lock_get_writer(lock) != RW_LOCK_NOT_LOCKED | |
1106 | - && i < SYNC_SPIN_ROUNDS) { | |
1107 | + while (i < SYNC_SPIN_ROUNDS | |
45532174 | 1108 | + && lock->lock_word != RW_LOCK_BIAS) { |
eccb488f ER |
1109 | if (srv_spin_wait_delay) { |
1110 | ut_delay(ut_rnd_interval(0, | |
1111 | srv_spin_wait_delay)); | |
45532174 | 1112 | @@ -485,9 +642,12 @@ |
eccb488f ER |
1113 | } else if (state == RW_LOCK_WAIT_EX) { |
1114 | ||
1115 | /* Spin waiting for the reader count field to become zero */ | |
1116 | - i = 0; | |
1117 | ||
1118 | +#ifdef HAVE_ATOMIC_BUILTINS | |
1119 | + while (lock->lock_word != RW_LOCK_BIAS | |
1120 | +#else | |
1121 | while (rw_lock_get_reader_count(lock) != 0 | |
1122 | +#endif | |
1123 | && i < SYNC_SPIN_ROUNDS) { | |
1124 | if (srv_spin_wait_delay) { | |
1125 | ut_delay(ut_rnd_interval(0, | |
45532174 | 1126 | @@ -500,7 +660,6 @@ |
eccb488f ER |
1127 | os_thread_yield(); |
1128 | } | |
1129 | } else { | |
1130 | - i = 0; /* Eliminate a compiler warning */ | |
1131 | ut_error; | |
1132 | } | |
1133 | ||
45532174 | 1134 | @@ -516,34 +675,69 @@ |
eccb488f ER |
1135 | /* We try once again to obtain the lock. Acquire the mutex protecting |
1136 | the rw-lock fields */ | |
1137 | ||
1138 | +#ifdef HAVE_ATOMIC_BUILTINS | |
1139 | + prev_state = state; | |
1140 | +#else | |
1141 | mutex_enter(rw_lock_get_mutex(lock)); | |
1142 | +#endif | |
1143 | ||
1144 | state = rw_lock_x_lock_low(lock, pass, file_name, line); | |
1145 | ||
1146 | +#ifdef HAVE_ATOMIC_BUILTINS | |
1147 | + if (state != prev_state) i=0; /* if progress, reset counter. */ | |
1148 | +#endif | |
1149 | + | |
1150 | if (state == RW_LOCK_EX) { | |
1151 | +#ifndef HAVE_ATOMIC_BUILTINS | |
1152 | mutex_exit(rw_lock_get_mutex(lock)); | |
1153 | +#endif | |
1154 | ||
1155 | return; /* Locking succeeded */ | |
1156 | } | |
45532174 | 1157 | |
eccb488f ER |
1158 | +#ifdef HAVE_ATOMIC_BUILTINS |
1159 | + /* like sync0sync.c doing */ | |
1160 | + i++; | |
1161 | + | |
1162 | + if (i < SYNC_SPIN_ROUNDS) { | |
1163 | + goto spin_loop; | |
1164 | + } | |
1165 | +#endif | |
45532174 | 1166 | + |
eccb488f ER |
1167 | rw_x_system_call_count++; |
1168 | ||
1169 | sync_array_reserve_cell(sync_primary_wait_array, | |
1170 | lock, | |
1171 | -#ifdef __WIN__ | |
1172 | - /* On windows RW_LOCK_WAIT_EX signifies | |
1173 | - that this thread should wait on the | |
1174 | - special wait_ex_event. */ | |
1175 | (state == RW_LOCK_WAIT_EX) | |
1176 | ? RW_LOCK_WAIT_EX : | |
1177 | -#endif | |
1178 | RW_LOCK_EX, | |
1179 | file_name, line, | |
1180 | &index); | |
1181 | ||
1182 | - rw_lock_set_waiters(lock, 1); | |
1183 | + if (state == RW_LOCK_WAIT_EX) { | |
1184 | + rw_lock_set_wx_waiters(lock, 1); | |
1185 | + } else { | |
1186 | + rw_lock_set_x_waiters(lock, 1); | |
1187 | + } | |
1188 | ||
1189 | +#ifdef HAVE_ATOMIC_BUILTINS | |
1190 | + /* like sync0sync.c doing */ | |
1191 | + for (i = 0; i < 4; i++) { | |
1192 | + prev_state = state; | |
1193 | + state = rw_lock_x_lock_low(lock, pass, file_name, line); | |
1194 | + if (state == RW_LOCK_EX) { | |
1195 | + sync_array_free_cell(sync_primary_wait_array, index); | |
1196 | + return; /* Locking succeeded */ | |
1197 | + } | |
1198 | + if (state != prev_state) { | |
1199 | + /* retry! */ | |
1200 | + sync_array_free_cell(sync_primary_wait_array, index); | |
1201 | + goto lock_loop; | |
1202 | + } | |
1203 | + } | |
1204 | +#else | |
1205 | mutex_exit(rw_lock_get_mutex(lock)); | |
1206 | +#endif | |
1207 | ||
1208 | if (srv_print_latch_waits) { | |
1209 | fprintf(stderr, | |
45532174 | 1210 | @@ -718,7 +912,9 @@ |
eccb488f ER |
1211 | ut_ad(lock); |
1212 | ut_ad(rw_lock_validate(lock)); | |
1213 | ||
1214 | +#ifndef HAVE_ATOMIC_BUILTINS | |
1215 | mutex_enter(&(lock->mutex)); | |
1216 | +#endif | |
1217 | ||
1218 | info = UT_LIST_GET_FIRST(lock->debug_list); | |
1219 | ||
45532174 | 1220 | @@ -728,7 +924,9 @@ |
eccb488f ER |
1221 | && (info->pass == 0) |
1222 | && (info->lock_type == lock_type)) { | |
1223 | ||
1224 | +#ifndef HAVE_ATOMIC_BUILTINS | |
1225 | mutex_exit(&(lock->mutex)); | |
1226 | +#endif | |
1227 | /* Found! */ | |
1228 | ||
1229 | return(TRUE); | |
45532174 | 1230 | @@ -736,7 +934,9 @@ |
eccb488f ER |
1231 | |
1232 | info = UT_LIST_GET_NEXT(list, info); | |
1233 | } | |
1234 | +#ifndef HAVE_ATOMIC_BUILTINS | |
1235 | mutex_exit(&(lock->mutex)); | |
1236 | +#endif | |
1237 | ||
1238 | return(FALSE); | |
1239 | } | |
45532174 | 1240 | @@ -758,21 +958,25 @@ |
eccb488f ER |
1241 | ut_ad(lock); |
1242 | ut_ad(rw_lock_validate(lock)); | |
1243 | ||
1244 | +#ifndef HAVE_ATOMIC_BUILTINS | |
1245 | mutex_enter(&(lock->mutex)); | |
1246 | +#endif | |
1247 | ||
1248 | if (lock_type == RW_LOCK_SHARED) { | |
1249 | if (lock->reader_count > 0) { | |
1250 | ret = TRUE; | |
1251 | } | |
1252 | } else if (lock_type == RW_LOCK_EX) { | |
1253 | - if (lock->writer == RW_LOCK_EX) { | |
1254 | + if (rw_lock_get_writer(lock) == RW_LOCK_EX) { | |
1255 | ret = TRUE; | |
1256 | } | |
1257 | } else { | |
1258 | ut_error; | |
1259 | } | |
1260 | ||
1261 | +#ifndef HAVE_ATOMIC_BUILTINS | |
1262 | mutex_exit(&(lock->mutex)); | |
1263 | +#endif | |
1264 | ||
1265 | return(ret); | |
1266 | } | |
45532174 | 1267 | @@ -801,16 +1005,26 @@ |
eccb488f ER |
1268 | |
1269 | count++; | |
1270 | ||
1271 | +#ifndef HAVE_ATOMIC_BUILTINS | |
1272 | mutex_enter(&(lock->mutex)); | |
1273 | +#endif | |
1274 | ||
1275 | if ((rw_lock_get_writer(lock) != RW_LOCK_NOT_LOCKED) | |
1276 | || (rw_lock_get_reader_count(lock) != 0) | |
1277 | - || (rw_lock_get_waiters(lock) != 0)) { | |
1278 | + || (rw_lock_get_s_waiters(lock) != 0) | |
1279 | + || (rw_lock_get_x_waiters(lock) != 0) | |
1280 | + || (rw_lock_get_wx_waiters(lock) != 0)) { | |
1281 | ||
1282 | fprintf(stderr, "RW-LOCK: %p ", lock); | |
1283 | ||
1284 | - if (rw_lock_get_waiters(lock)) { | |
1285 | - fputs(" Waiters for the lock exist\n", stderr); | |
1286 | + if (rw_lock_get_s_waiters(lock)) { | |
1287 | + fputs(" s_waiters for the lock exist,", stderr); | |
1288 | + } | |
1289 | + if (rw_lock_get_x_waiters(lock)) { | |
1290 | + fputs(" x_waiters for the lock exist\n", stderr); | |
1291 | + } | |
1292 | + if (rw_lock_get_wx_waiters(lock)) { | |
1293 | + fputs(" wait_ex_waiters for the lock exist\n", stderr); | |
1294 | } else { | |
1295 | putc('\n', stderr); | |
1296 | } | |
45532174 | 1297 | @@ -822,7 +1036,9 @@ |
eccb488f ER |
1298 | } |
1299 | } | |
1300 | ||
1301 | +#ifndef HAVE_ATOMIC_BUILTINS | |
1302 | mutex_exit(&(lock->mutex)); | |
1303 | +#endif | |
1304 | lock = UT_LIST_GET_NEXT(list, lock); | |
1305 | } | |
1306 | ||
45532174 | 1307 | @@ -847,10 +1063,18 @@ |
eccb488f ER |
1308 | |
1309 | if ((rw_lock_get_writer(lock) != RW_LOCK_NOT_LOCKED) | |
1310 | || (rw_lock_get_reader_count(lock) != 0) | |
1311 | - || (rw_lock_get_waiters(lock) != 0)) { | |
1312 | + || (rw_lock_get_s_waiters(lock) != 0) | |
1313 | + || (rw_lock_get_x_waiters(lock) != 0) | |
1314 | + || (rw_lock_get_wx_waiters(lock) != 0)) { | |
1315 | ||
1316 | - if (rw_lock_get_waiters(lock)) { | |
1317 | - fputs(" Waiters for the lock exist\n", stderr); | |
1318 | + if (rw_lock_get_s_waiters(lock)) { | |
1319 | + fputs(" s_waiters for the lock exist,", stderr); | |
1320 | + } | |
1321 | + if (rw_lock_get_x_waiters(lock)) { | |
1322 | + fputs(" x_waiters for the lock exist\n", stderr); | |
1323 | + } | |
1324 | + if (rw_lock_get_wx_waiters(lock)) { | |
1325 | + fputs(" wait_ex_waiters for the lock exist\n", stderr); | |
1326 | } else { | |
1327 | putc('\n', stderr); | |
1328 | } | |
45532174 | 1329 | @@ -909,14 +1133,18 @@ |
eccb488f ER |
1330 | lock = UT_LIST_GET_FIRST(rw_lock_list); |
1331 | ||
1332 | while (lock != NULL) { | |
1333 | +#ifndef HAVE_ATOMIC_BUILTINS | |
1334 | mutex_enter(rw_lock_get_mutex(lock)); | |
1335 | +#endif | |
1336 | ||
1337 | if ((rw_lock_get_writer(lock) != RW_LOCK_NOT_LOCKED) | |
1338 | || (rw_lock_get_reader_count(lock) != 0)) { | |
1339 | count++; | |
1340 | } | |
1341 | ||
1342 | +#ifndef HAVE_ATOMIC_BUILTINS | |
1343 | mutex_exit(rw_lock_get_mutex(lock)); | |
1344 | +#endif | |
1345 | lock = UT_LIST_GET_NEXT(list, lock); | |
1346 | } | |
1347 | ||
45532174 ER |
1348 | diff -ruN a/patch_info/innodb_rw_lock.info b/patch_info/innodb_rw_lock.info |
1349 | --- /dev/null 1970-01-01 09:00:00.000000000 +0900 | |
1350 | +++ b/patch_info/innodb_rw_lock.info 2009-04-16 16:15:28.000000000 +0900 | |
eccb488f ER |
1351 | @@ -0,0 +1,6 @@ |
1352 | +File=innodb_rw_lock.patch | |
1353 | +Name=Fix of InnoDB rw_locks | |
1354 | +Version=1.0 | |
1355 | +Author=Yasufumi Kinoshita | |
1356 | +License=BSD | |
1357 | +Comment= |