]> git.pld-linux.org Git - packages/mysql.git/blame - mysql-innodb_rw_lock.patch
projects / packages / mysql.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
move status action to status function
[packages/mysql.git] / mysql-innodb_rw_lock.patch
CommitLineData
4b771f0e
ER		 1diff -ruN a/innobase/btr/btr0cur.c b/innobase/btr/btr0cur.c
2--- a/innobase/btr/btr0cur.c 2009-10-22 15:15:05.000000000 +0900
3+++ b/innobase/btr/btr0cur.c 2009-10-22 15:18:44.000000000 +0900
4@@ -313,7 +313,7 @@
5 #ifdef UNIV_SEARCH_PERF_STAT
6 info->n_searches++;
7 #endif
8- if (btr_search_latch.writer == RW_LOCK_NOT_LOCKED
9+ if (rw_lock_get_writer(&btr_search_latch) == RW_LOCK_NOT_LOCKED
10 && latch_mode <= BTR_MODIFY_LEAF && info->last_hash_succ
11 && !estimate
12 #ifdef PAGE_CUR_LE_OR_EXTENDS
45532174 13diff -ruN a/innobase/btr/btr0sea.c b/innobase/btr/btr0sea.c
4b771f0e
ER		 14--- a/innobase/btr/btr0sea.c 2009-10-22 15:15:05.000000000 +0900
15+++ b/innobase/btr/btr0sea.c 2009-10-22 15:18:44.000000000 +0900
16@@ -773,8 +773,8 @@
45532174
ER		 17 rw_lock_s_lock(&btr_search_latch);
18 }
eccb488f 19
45532174 20- ut_ad(btr_search_latch.writer != RW_LOCK_EX);
4b771f0e
ER		 21- ut_ad(btr_search_latch.reader_count > 0);
22+ ut_ad(rw_lock_get_writer(&btr_search_latch) != RW_LOCK_EX);
23+ ut_ad(rw_lock_get_reader_count(&btr_search_latch) > 0);
45532174
ER		 24
25 rec = ha_search_and_get_data(btr_search_sys->hash_index, fold);
eccb488f 26
4b771f0e
ER		 27diff -ruN a/innobase/buf/buf0buf.c b/innobase/buf/buf0buf.c
28--- a/innobase/buf/buf0buf.c 2009-10-22 15:15:05.000000000 +0900
29+++ b/innobase/buf/buf0buf.c 2009-10-22 15:18:44.000000000 +0900
30@@ -1292,7 +1292,7 @@
31
32 if (mode == BUF_GET_NOWAIT) {
33 if (rw_latch == RW_S_LATCH) {
34- success = rw_lock_s_lock_func_nowait(&(block->lock),
35+ success = rw_lock_s_lock_nowait(&(block->lock),
36 file, line);
37 fix_type = MTR_MEMO_PAGE_S_FIX;
38 } else {
39@@ -1442,7 +1442,7 @@
40 ut_ad(!ibuf_inside() || ibuf_page(block->space, block->offset));
41
42 if (rw_latch == RW_S_LATCH) {
43- success = rw_lock_s_lock_func_nowait(&(block->lock),
44+ success = rw_lock_s_lock_nowait(&(block->lock),
45 file, line);
46 fix_type = MTR_MEMO_PAGE_S_FIX;
47 } else {
48@@ -1596,7 +1596,7 @@
49 ut_ad(!ibuf_inside() || (mode == BUF_KEEP_OLD));
50
51 if (rw_latch == RW_S_LATCH) {
52- success = rw_lock_s_lock_func_nowait(&(block->lock),
53+ success = rw_lock_s_lock_nowait(&(block->lock),
54 file, line);
55 fix_type = MTR_MEMO_PAGE_S_FIX;
56 } else {
57diff -ruN a/innobase/include/buf0buf.ic b/innobase/include/buf0buf.ic
58--- a/innobase/include/buf0buf.ic 2009-10-22 15:15:05.000000000 +0900
59+++ b/innobase/include/buf0buf.ic 2009-10-22 16:12:25.000000000 +0900
60@@ -523,7 +523,7 @@
61 #ifdef UNIV_SYNC_DEBUG
62 ibool ret;
63
64- ret = rw_lock_s_lock_func_nowait(&(block->debug_latch), file, line);
65+ ret = rw_lock_s_lock_nowait(&(block->debug_latch), file, line);
66
67 ut_ad(ret == TRUE);
68 ut_ad(mutex_own(&block->mutex));
69diff -ruN a/innobase/include/os0sync.h b/innobase/include/os0sync.h
70--- a/innobase/include/os0sync.h 2009-09-10 04:02:59.000000000 +0900
71+++ b/innobase/include/os0sync.h 2009-10-22 15:18:44.000000000 +0900
72@@ -1,11 +1,35 @@
73+/*****************************************************************************
eccb488f 74+
4b771f0e
ER		 75+Copyright (c) 1995, 2009, Innobase Oy. All Rights Reserved.
76+Copyright (c) 2008, Google Inc.
77+
78+Portions of this file contain modifications contributed and copyrighted by
79+Google, Inc. Those modifications are gratefully acknowledged and are described
80+briefly in the InnoDB documentation. The contributions by Google are
81+incorporated with their permission, and subject to the conditions contained in
82+the file COPYING.Google.
83+
84+This program is free software; you can redistribute it and/or modify it under
85+the terms of the GNU General Public License as published by the Free Software
86+Foundation; version 2 of the License.
87+
88+This program is distributed in the hope that it will be useful, but WITHOUT
89+ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
90+FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
91+
92+You should have received a copy of the GNU General Public License along with
93+this program; if not, write to the Free Software Foundation, Inc., 59 Temple
94+Place, Suite 330, Boston, MA 02111-1307 USA
95+
96+*****************************************************************************/
97+
98 /******************************************************
99 The interface to the operating system
100 synchronization primitives.
eccb488f 101
4b771f0e 102-(c) 1995 Innobase Oy
eccb488f 103-
4b771f0e
ER		 104 Created 9/6/1995 Heikki Tuuri
105 *******************************************************/
eccb488f 106+
4b771f0e
ER		 107 #ifndef os0sync_h
108 #define os0sync_h
109
110@@ -261,6 +285,23 @@
111 /*===============*/
112 os_fast_mutex_t* fast_mutex); /* in: mutex to free */
113
eccb488f 114+#ifdef HAVE_ATOMIC_BUILTINS
4b771f0e
ER		 115+/**************************************************************
116+Atomic compare-and-swap for InnoDB. Currently requires GCC atomic builtins.
117+Returns true if swapped, ptr is pointer to target, old_val is value to
118+compare to, new_val is the value to swap in. */
119+#define os_compare_and_swap(ptr, old_val, new_val) \
120+ __sync_bool_compare_and_swap(ptr, old_val, new_val)
121+
122+/**************************************************************
123+Atomic increment for InnoDB. Currently requires GCC atomic builtins.
124+Returns the resulting value, ptr is pointer to target, amount is the
125+amount of increment. */
126+#define os_atomic_increment(ptr, amount) \
127+ __sync_add_and_fetch(ptr, amount)
128+
129+#endif /* HAVE_ATOMIC_BUILTINS */
130+
131 #ifndef UNIV_NONINL
132 #include "os0sync.ic"
eccb488f 133 #endif
4b771f0e
ER		 134diff -ruN a/innobase/include/sync0rw.h b/innobase/include/sync0rw.h
135--- a/innobase/include/sync0rw.h 2009-09-10 04:02:59.000000000 +0900
136+++ b/innobase/include/sync0rw.h 2009-10-22 15:18:44.000000000 +0900
137@@ -1,8 +1,31 @@
138+/*****************************************************************************
139+
140+Copyright (c) 1995, 2009, Innobase Oy. All Rights Reserved.
141+Copyright (c) 2008, Google Inc.
142+
143+Portions of this file contain modifications contributed and copyrighted by
144+Google, Inc. Those modifications are gratefully acknowledged and are described
145+briefly in the InnoDB documentation. The contributions by Google are
146+incorporated with their permission, and subject to the conditions contained in
147+the file COPYING.Google.
148+
149+This program is free software; you can redistribute it and/or modify it under
150+the terms of the GNU General Public License as published by the Free Software
151+Foundation; version 2 of the License.
152+
153+This program is distributed in the hope that it will be useful, but WITHOUT
154+ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
155+FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
156+
157+You should have received a copy of the GNU General Public License along with
158+this program; if not, write to the Free Software Foundation, Inc., 59 Temple
159+Place, Suite 330, Boston, MA 02111-1307 USA
160+
161+*****************************************************************************/
162+
163 /******************************************************
164 The read-write lock (for threads, not for database transactions)
165
166-(c) 1995 Innobase Oy
167-
168 Created 9/11/1995 Heikki Tuuri
169 *******************************************************/
170
171@@ -24,6 +47,12 @@
172 #define RW_X_LATCH 2
173 #define RW_NO_LATCH 3
174
175+/* We decrement lock_word by this amount for each x_lock. It is also the
176+start value for the lock_word, meaning that it limits the maximum number
177+of concurrent read locks before the rw_lock breaks. The current value of
178+0x00100000 allows 1,048,575 concurrent readers and 2047 recursive writers.*/
179+#define X_LOCK_DECR 0x00100000
180+
181 typedef struct rw_lock_struct rw_lock_t;
182 #ifdef UNIV_SYNC_DEBUG
183 typedef struct rw_lock_debug_struct rw_lock_debug_t;
184@@ -47,14 +76,14 @@
185 there may be waiters for the event */
186 #endif /* UNIV_SYNC_DEBUG */
187
188-extern ulint rw_s_system_call_count;
189-extern ulint rw_s_spin_wait_count;
190-extern ulint rw_s_exit_count;
191-extern ulint rw_s_os_wait_count;
192-extern ulint rw_x_system_call_count;
193-extern ulint rw_x_spin_wait_count;
194-extern ulint rw_x_os_wait_count;
195-extern ulint rw_x_exit_count;
196+extern ib_longlong rw_s_spin_wait_count;
197+extern ib_longlong rw_s_spin_round_count;
198+extern ib_longlong rw_s_exit_count;
199+extern ib_longlong rw_s_os_wait_count;
200+extern ib_longlong rw_x_spin_wait_count;
201+extern ib_longlong rw_x_spin_round_count;
202+extern ib_longlong rw_x_os_wait_count;
203+extern ib_longlong rw_x_exit_count;
204
205 /**********************************************************************
206 Creates, or rather, initializes an rw-lock object in a specified memory
207@@ -116,8 +145,22 @@
208 NOTE! The following macros should be used in rw s-locking, not the
209 corresponding function. */
210
211-#define rw_lock_s_lock_nowait(M) rw_lock_s_lock_func_nowait(\
212- (M), __FILE__, __LINE__)
213+#define rw_lock_s_lock_nowait(M, F, L) rw_lock_s_lock_low(\
214+ (M), 0, (F), (L))
215+/**********************************************************************
216+Low-level function which tries to lock an rw-lock in s-mode. Performs no
217+spinning. */
218+UNIV_INLINE
219+ibool
220+rw_lock_s_lock_low(
221+/*===============*/
222+ /* out: TRUE if success */
223+ rw_lock_t* lock, /* in: pointer to rw-lock */
224+ ulint pass __attribute__((unused)),
225+ /* in: pass value; != 0, if the lock will be
226+ passed to another thread to unlock */
227+ const char* file_name, /* in: file name where lock requested */
228+ ulint line); /* in: line where requested */
229 /**********************************************************************
230 NOTE! Use the corresponding macro, not directly this function, except if
231 you supply the file name and line number. Lock an rw-lock in shared mode
232@@ -135,18 +178,6 @@
233 const char* file_name,/* in: file name where lock requested */
234 ulint line); /* in: line where requested */
235 /**********************************************************************
236-NOTE! Use the corresponding macro, not directly this function, except if
237-you supply the file name and line number. Lock an rw-lock in shared mode
238-for the current thread if the lock can be acquired immediately. */
239-UNIV_INLINE
240-ibool
241-rw_lock_s_lock_func_nowait(
242-/*=======================*/
243- /* out: TRUE if success */
244- rw_lock_t* lock, /* in: pointer to rw-lock */
245- const char* file_name,/* in: file name where lock requested */
246- ulint line); /* in: line where requested */
247-/**********************************************************************
248 NOTE! Use the corresponding macro, not directly this function! Lock an
249 rw-lock in exclusive mode for the current thread if the lock can be
250 obtained immediately. */
251@@ -338,6 +369,41 @@
252 rw_lock_get_reader_count(
253 /*=====================*/
254 rw_lock_t* lock);
255+/**********************************************************************
256+Decrements lock_word the specified amount if it is greater than 0.
257+This is used by both s_lock and x_lock operations. */
258+UNIV_INLINE
259+ibool
260+rw_lock_lock_word_decr(
261+/*===================*/
262+ /* out: TRUE if decr occurs */
263+ rw_lock_t* lock, /* in: rw-lock */
264+ ulint amount); /* in: amount to decrement */
265+/**********************************************************************
266+Increments lock_word the specified amount and returns new value. */
267+UNIV_INLINE
268+lint
269+rw_lock_lock_word_incr(
270+/*===================*/
271+ /* out: TRUE if decr occurs */
272+ rw_lock_t* lock,
273+ ulint amount); /* in: rw-lock */
274+/**********************************************************************
275+This function sets the lock->writer_thread and lock->recursive fields.
276+For platforms where we are using atomic builtins instead of lock->mutex
277+it sets the lock->writer_thread field using atomics to ensure memory
278+ordering. Note that it is assumed that the caller of this function
279+effectively owns the lock i.e.: nobody else is allowed to modify
280+lock->writer_thread at this point in time.
281+The protocol is that lock->writer_thread MUST be updated BEFORE the
282+lock->recursive flag is set. */
283+UNIV_INLINE
284+void
285+rw_lock_set_writer_id_and_recursion_flag(
286+/*=====================================*/
287+ rw_lock_t* lock, /* in/out: lock to work on */
288+ ibool recursive); /* in: TRUE if recursion
289+ allowed */
290 #ifdef UNIV_SYNC_DEBUG
291 /**********************************************************************
292 Checks if the thread has locked the rw-lock in the specified mode, with
293@@ -417,47 +483,33 @@
294 field. Then no new readers are allowed in. */
eccb488f 295
4b771f0e
ER		 296 struct rw_lock_struct {
297+ volatile lint lock_word;
298+ /* Holds the state of the lock. */
299+ volatile ulint waiters;/* 1: there are waiters */
300+ volatile ibool recursive;/* Default value FALSE which means the lock
301+ is non-recursive. The value is typically set
302+ to TRUE making normal rw_locks recursive. In
303+ case of asynchronous IO, when a non-zero
304+ value of 'pass' is passed then we keep the
305+ lock non-recursive.
306+ This flag also tells us about the state of
307+ writer_thread field. If this flag is set
308+ then writer_thread MUST contain the thread
309+ id of the current x-holder or wait-x thread.
310+ This flag must be reset in x_unlock
311+ functions before incrementing the lock_word */
312+ volatile os_thread_id_t writer_thread;
313+ /* Thread id of writer thread. Is only
314+ guaranteed to have sane and non-stale
315+ value iff recursive flag is set. */
316 os_event_t event; /* Used by sync0arr.c for thread queueing */
317-
318-#ifdef __WIN__
319- os_event_t wait_ex_event; /* This windows specific event is
320- used by the thread which has set the
321- lock state to RW_LOCK_WAIT_EX. The
322- rw_lock design guarantees that this
323- thread will be the next one to proceed
324- once the current the event gets
325- signalled. See LEMMA 2 in sync0sync.c */
326-#endif
327-
eccb488f 328- ulint reader_count; /* Number of readers who have locked this
4b771f0e 329- lock in the shared mode */
eccb488f 330- ulint writer; /* This field is set to RW_LOCK_EX if there
4b771f0e
ER		 331- is a writer owning the lock (in exclusive
332- mode), RW_LOCK_WAIT_EX if a writer is
333- queueing for the lock, and
334- RW_LOCK_NOT_LOCKED, otherwise. */
eccb488f 335- os_thread_id_t writer_thread;
4b771f0e 336- /* Thread id of a possible writer thread */
eccb488f 337- ulint writer_count; /* Number of times the same thread has
4b771f0e
ER		 338- recursively locked the lock in the exclusive
339- mode */
340+ os_event_t wait_ex_event;
341+ /* Event for next-writer to wait on. A thread
342+ must decrement lock_word before waiting. */
eccb488f
ER		 343+#ifndef HAVE_ATOMIC_BUILTINS
344 mutex_t mutex; /* The mutex protecting rw_lock_struct */
4b771f0e
ER		 345- ulint pass; /* Default value 0. This is set to some
346- value != 0 given by the caller of an x-lock
347- operation, if the x-lock is to be passed to
348- another thread to unlock (which happens in
349- asynchronous i/o). */
eccb488f
ER		 350- ulint waiters; /* This ulint is set to 1 if there are
351- waiters (readers or writers) in the global
352- wait array, waiting for this rw_lock.
353- Otherwise, == 0. */
354- ibool writer_is_wait_ex;
4b771f0e
ER		 355- /* This is TRUE if the writer field is
356- RW_LOCK_WAIT_EX; this field is located far
357- from the memory update hotspot fields which
358- are at the start of this struct, thus we can
359- peek this field without causing much memory
360- bus traffic */
361+#endif /* HAVE_ATOMIC_BUILTINS */
362+
363 UT_LIST_NODE_T(rw_lock_t) list;
364 /* All allocated rw locks are put into a
365 list */
366@@ -465,15 +517,23 @@
367 UT_LIST_BASE_NODE_T(rw_lock_debug_t) debug_list;
368 /* In the debug version: pointer to the debug
369 info list of the lock */
370+ ulint level; /* Level in the global latching order. */
371 #endif /* UNIV_SYNC_DEBUG */
372- ulint level; /* Level in the global latching
373- order; default SYNC_LEVEL_NONE */
374+ ulint count_os_wait; /* Count of os_waits. May not be accurate */
375 const char* cfile_name;/* File name where lock created */
376- ulint cline; /* Line where created */
377+ /* last s-lock file/line is not guaranteed to be correct */
378 const char* last_s_file_name;/* File name where last s-locked */
379 const char* last_x_file_name;/* File name where last x-locked */
380- ulint last_s_line; /* Line number where last time s-locked */
381- ulint last_x_line; /* Line number where last time x-locked */
382+ ibool writer_is_wait_ex;
383+ /* This is TRUE if the writer field is
384+ RW_LOCK_WAIT_EX; this field is located far
385+ from the memory update hotspot fields which
386+ are at the start of this struct, thus we can
387+ peek this field without causing much memory
388+ bus traffic */
389+ unsigned cline:14; /* Line where created */
390+ unsigned last_s_line:14; /* Line number where last time s-locked */
391+ unsigned last_x_line:14; /* Line number where last time x-locked */
392 ulint magic_n;
393 };
394
45532174 395diff -ruN a/innobase/include/sync0rw.ic b/innobase/include/sync0rw.ic
4b771f0e
ER		 396--- a/innobase/include/sync0rw.ic 2009-09-10 04:02:59.000000000 +0900
397+++ b/innobase/include/sync0rw.ic 2009-10-22 15:18:44.000000000 +0900
398@@ -1,8 +1,31 @@
399+/*****************************************************************************
400+
401+Copyright (c) 1995, 2009, Innobase Oy. All Rights Reserved.
402+Copyright (c) 2008, Google Inc.
403+
404+Portions of this file contain modifications contributed and copyrighted by
405+Google, Inc. Those modifications are gratefully acknowledged and are described
406+briefly in the InnoDB documentation. The contributions by Google are
407+incorporated with their permission, and subject to the conditions contained in
408+the file COPYING.Google.
409+
410+This program is free software; you can redistribute it and/or modify it under
411+the terms of the GNU General Public License as published by the Free Software
412+Foundation; version 2 of the License.
413+
414+This program is distributed in the hope that it will be useful, but WITHOUT
415+ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
416+FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
417+
418+You should have received a copy of the GNU General Public License along with
419+this program; if not, write to the Free Software Foundation, Inc., 59 Temple
420+Place, Suite 330, Boston, MA 02111-1307 USA
421+
422+*****************************************************************************/
423+
424 /******************************************************
425 The read-write lock (for threads)
426
427-(c) 1995 Innobase Oy
428-
429 Created 9/11/1995 Heikki Tuuri
430 *******************************************************/
431
432@@ -49,53 +72,88 @@
eccb488f 433 ulint
4b771f0e 434 rw_lock_get_waiters(
eccb488f 435 /*================*/
4b771f0e
ER		 436- rw_lock_t* lock)
437+ /* out: 1 if waiters, 0 otherwise */
438+ rw_lock_t* lock) /* in: rw-lock */
eccb488f 439 {
4b771f0e 440 return(lock->waiters);
45532174 441 }
4b771f0e
ER		 442+
443+/************************************************************************
444+Sets lock->waiters to 1. It is not an error if lock->waiters is already
445+1. On platforms where ATOMIC builtins are used this function enforces a
446+memory barrier. */
45532174 447 UNIV_INLINE
4b771f0e 448 void
45532174 449-rw_lock_set_waiters(
4b771f0e
ER		 450-/*================*/
451- rw_lock_t* lock,
452- ulint flag)
453+rw_lock_set_waiter_flag(
454+/*====================*/
455+ rw_lock_t* lock) /* in: rw-lock */
eccb488f
ER		 456 {
457- lock->waiters = flag;
45532174 458+#ifdef HAVE_ATOMIC_BUILTINS
4b771f0e
ER		 459+ os_compare_and_swap(&lock->waiters, 0, 1);
460+#else /* HAVE_ATOMIC_BUILTINS */
461+ lock->waiters = 1;
462+#endif /* HAVE_ATOMIC_BUILTINS */
463 }
464+
465+/************************************************************************
466+Resets lock->waiters to 0. It is not an error if lock->waiters is already
467+0. On platforms where ATOMIC builtins are used this function enforces a
468+memory barrier. */
469 UNIV_INLINE
470-ulint
471-rw_lock_get_writer(
472-/*===============*/
473- rw_lock_t* lock)
eccb488f 474+void
4b771f0e
ER		 475+rw_lock_reset_waiter_flag(
476+/*======================*/
477+ rw_lock_t* lock) /* in: rw-lock */
478 {
479- return(lock->writer);
45532174 480+#ifdef HAVE_ATOMIC_BUILTINS
4b771f0e
ER		 481+ os_compare_and_swap(&lock->waiters, 1, 0);
482+#else /* HAVE_ATOMIC_BUILTINS */
483+ lock->waiters = 0;
484+#endif /* HAVE_ATOMIC_BUILTINS */
eccb488f 485 }
4b771f0e
ER		 486+
487+/**********************************************************************
488+Returns the write-status of the lock - this function made more sense
489+with the old rw_lock implementation. */
eccb488f 490 UNIV_INLINE
4b771f0e
ER		 491-void
492-rw_lock_set_writer(
493+ulint
494+rw_lock_get_writer(
eccb488f 495 /*===============*/
4b771f0e
ER		 496- rw_lock_t* lock,
497- ulint flag)
498+ rw_lock_t* lock)
eccb488f 499 {
4b771f0e
ER		 500- lock->writer = flag;
501+ lint lock_word = lock->lock_word;
502+ if(lock_word > 0) {
503+ /* return NOT_LOCKED in s-lock state, like the writer
504+ member of the old lock implementation. */
eccb488f 505+ return(RW_LOCK_NOT_LOCKED);
4b771f0e 506+ } else if (((-lock_word) % X_LOCK_DECR) == 0) {
eccb488f 507+ return(RW_LOCK_EX);
4b771f0e
ER		 508+ } else {
509+ ut_ad(lock_word > -X_LOCK_DECR);
510+ return(RW_LOCK_WAIT_EX);
eccb488f 511+ }
eccb488f 512 }
4b771f0e
ER		 513+
514+/**********************************************************************
515+Returns number of readers. */
eccb488f 516 UNIV_INLINE
4b771f0e
ER		 517 ulint
518 rw_lock_get_reader_count(
519 /*=====================*/
520 rw_lock_t* lock)
eccb488f 521 {
4b771f0e
ER		 522- return(lock->reader_count);
523-}
524-UNIV_INLINE
525-void
526-rw_lock_set_reader_count(
527-/*=====================*/
528- rw_lock_t* lock,
529- ulint count)
530-{
531- lock->reader_count = count;
532+ lint lock_word = lock->lock_word;
533+ if(lock_word > 0) {
534+ /* s-locked, no x-waiters */
535+ return(X_LOCK_DECR - lock_word);
536+ } else if (lock_word < 0 && lock_word > -X_LOCK_DECR) {
537+ /* s-locked, with x-waiters */
538+ return((ulint)(-lock_word));
539+ }
540+ return(0);
eccb488f 541 }
4b771f0e 542+
eccb488f
ER		 543+#ifndef HAVE_ATOMIC_BUILTINS
544 UNIV_INLINE
545 mutex_t*
546 rw_lock_get_mutex(
4b771f0e 547@@ -104,6 +162,7 @@
eccb488f
ER		 548 {
549 return(&(lock->mutex));
550 }
551+#endif
552
553 /**********************************************************************
554 Returns the value of writer_count for the lock. Does not reserve the lock
4b771f0e
ER		 555@@ -115,7 +174,126 @@
556 /* out: value of writer_count */
557 rw_lock_t* lock) /* in: rw-lock */
558 {
559- return(lock->writer_count);
560+ lint lock_copy = lock->lock_word;
561+ /* If there is a reader, lock_word is not divisible by X_LOCK_DECR */
562+ if(lock_copy > 0 || (-lock_copy) % X_LOCK_DECR != 0) {
563+ return(0);
564+ }
565+ return(((-lock_copy) / X_LOCK_DECR) + 1);
566+}
567+
568+/**********************************************************************
569+Two different implementations for decrementing the lock_word of a rw_lock:
570+one for systems supporting atomic operations, one for others. This does
571+does not support recusive x-locks: they should be handled by the caller and
572+need not be atomic since they are performed by the current lock holder.
573+Returns true if the decrement was made, false if not. */
574+UNIV_INLINE
575+ibool
576+rw_lock_lock_word_decr(
577+/*===================*/
578+ /* out: TRUE if decr occurs */
579+ rw_lock_t* lock, /* in: rw-lock */
580+ ulint amount) /* in: amount of decrement */
581+{
582+
583+#ifdef HAVE_ATOMIC_BUILTINS
584+
585+ lint local_lock_word = lock->lock_word;
586+ while (local_lock_word > 0) {
587+ if(os_compare_and_swap(&(lock->lock_word),
588+ local_lock_word,
589+ local_lock_word - amount)) {
590+ return(TRUE);
591+ }
592+ local_lock_word = lock->lock_word;
593+ }
594+ return(FALSE);
595+
596+#else /* HAVE_ATOMIC_BUILTINS */
597+
598+ ibool success = FALSE;
599+ mutex_enter(&(lock->mutex));
600+ if(lock->lock_word > 0) {
601+ lock->lock_word -= amount;
602+ success = TRUE;
603+ }
604+ mutex_exit(&(lock->mutex));
605+ return(success);
606+
607+#endif /* HAVE_ATOMIC_BUILTINS */
608+}
609+
610+/**********************************************************************
611+Two different implementations for incrementing the lock_word of a rw_lock:
612+one for systems supporting atomic operations, one for others.
613+Returns the value of lock_word after increment. */
614+UNIV_INLINE
615+lint
616+rw_lock_lock_word_incr(
617+/*===================*/
618+ /* out: lock->lock_word after increment */
619+ rw_lock_t* lock, /* in: rw-lock */
620+ ulint amount) /* in: amount of increment */
621+{
622+
623+#ifdef HAVE_ATOMIC_BUILTINS
624+
625+ return(os_atomic_increment(&(lock->lock_word), amount));
626+
627+#else /* HAVE_ATOMIC_BUILTINS */
628+
629+ lint local_lock_word;
630+
631+ mutex_enter(&(lock->mutex));
632+
633+ lock->lock_word += amount;
634+ local_lock_word = lock->lock_word;
635+
636+ mutex_exit(&(lock->mutex));
637+
638+ return(local_lock_word);
639+
640+#endif /* HAVE_ATOMIC_BUILTINS */
641+}
642+
643+/**********************************************************************
644+This function sets the lock->writer_thread and lock->recursive fields.
645+For platforms where we are using atomic builtins instead of lock->mutex
646+it sets the lock->writer_thread field using atomics to ensure memory
647+ordering. Note that it is assumed that the caller of this function
648+effectively owns the lock i.e.: nobody else is allowed to modify
649+lock->writer_thread at this point in time.
650+The protocol is that lock->writer_thread MUST be updated BEFORE the
651+lock->recursive flag is set. */
652+UNIV_INLINE
653+void
654+rw_lock_set_writer_id_and_recursion_flag(
655+/*=====================================*/
656+ rw_lock_t* lock, /* in/out: lock to work on */
657+ ibool recursive) /* in: TRUE if recursion
658+ allowed */
659+{
660+ os_thread_id_t curr_thread = os_thread_get_curr_id();
661+
662+#ifdef HAVE_ATOMIC_BUILTINS
663+ os_thread_id_t local_thread;
664+ ibool success;
665+
666+ local_thread = lock->writer_thread;
667+ success = os_compare_and_swap(&lock->writer_thread,
668+ local_thread, curr_thread);
669+ ut_a(success);
670+ lock->recursive = recursive;
671+
672+#else /* HAVE_ATOMIC_BUILTINS */
673+
674+ mutex_enter(&lock->mutex);
675+ lock->writer_thread = curr_thread;
676+ lock->recursive = recursive;
677+ mutex_exit(&lock->mutex);
678+
679+#endif /* HAVE_ATOMIC_BUILTINS */
680 }
681
682 /**********************************************************************
683@@ -133,26 +311,21 @@
eccb488f
ER		 684 const char* file_name, /* in: file name where lock requested */
685 ulint line) /* in: line where requested */
686 {
687-#ifdef UNIV_SYNC_DEBUG
4b771f0e
ER		 688- ut_ad(mutex_own(rw_lock_get_mutex(lock)));
689-#endif /* UNIV_SYNC_DEBUG */
690- /* Check if the writer field is free */
691-
692- if (UNIV_LIKELY(lock->writer == RW_LOCK_NOT_LOCKED)) {
693- /* Set the shared lock by incrementing the reader count */
694- lock->reader_count++;
695+ /* TODO: study performance of UNIV_LIKELY branch prediction hints. */
696+ if (!rw_lock_lock_word_decr(lock, 1)) {
697+ /* Locking did not succeed */
698+ return(FALSE);
699+ }
eccb488f
ER		 700
701 #ifdef UNIV_SYNC_DEBUG
4b771f0e
ER		 702- rw_lock_add_debug_info(lock, pass, RW_LOCK_SHARED, file_name,
703- line);
704+ rw_lock_add_debug_info(lock, pass, RW_LOCK_SHARED, file_name, line);
705 #endif
706- lock->last_s_file_name = file_name;
707- lock->last_s_line = line;
708-
709- return(TRUE); /* locking succeeded */
710- }
711+ /* These debugging values are not set safely: they may be incorrect
712+ or even refer to a line that is invalid for the file name. */
713+ lock->last_s_file_name = file_name;
714+ lock->last_s_line = line;
715
716- return(FALSE); /* locking did not succeed */
717+ return(TRUE); /* locking succeeded */
718 }
719
720 /**********************************************************************
721@@ -167,11 +340,10 @@
eccb488f
ER		 722 const char* file_name, /* in: file name where requested */
723 ulint line) /* in: line where lock requested */
724 {
725- ut_ad(lock->writer == RW_LOCK_NOT_LOCKED);
4b771f0e
ER		 726- ut_ad(rw_lock_get_reader_count(lock) == 0);
727+ ut_ad(lock->lock_word == X_LOCK_DECR);
eccb488f 728
4b771f0e
ER		 729- /* Set the shared lock by incrementing the reader count */
730- lock->reader_count++;
731+ /* Indicate there is a new reader by decrementing lock_word */
732+ lock->lock_word--;
eccb488f
ER		 733
734 lock->last_s_file_name = file_name;
735 lock->last_s_line = line;
4b771f0e
ER		 736@@ -194,13 +366,11 @@
737 ulint line) /* in: line where lock requested */
738 {
739 ut_ad(rw_lock_validate(lock));
740- ut_ad(rw_lock_get_reader_count(lock) == 0);
741- ut_ad(rw_lock_get_writer(lock) == RW_LOCK_NOT_LOCKED);
742+ ut_ad(lock->lock_word == X_LOCK_DECR);
eccb488f 743
4b771f0e
ER		 744- rw_lock_set_writer(lock, RW_LOCK_EX);
745+ lock->lock_word -= X_LOCK_DECR;
eccb488f 746 lock->writer_thread = os_thread_get_curr_id();
4b771f0e
ER		 747- lock->writer_count++;
748- lock->pass = 0;
749+ lock->recursive = TRUE;
eccb488f
ER		 750
751 lock->last_x_file_name = file_name;
4b771f0e
ER		 752 lock->last_x_line = line;
753@@ -241,15 +411,12 @@
eccb488f
ER		 754 ut_ad(!rw_lock_own(lock, RW_LOCK_SHARED)); /* see NOTE above */
755 #endif /* UNIV_SYNC_DEBUG */
756
4b771f0e
ER		 757- mutex_enter(rw_lock_get_mutex(lock));
758-
759- if (UNIV_LIKELY(rw_lock_s_lock_low(lock, pass, file_name, line))) {
760- mutex_exit(rw_lock_get_mutex(lock));
761+ /* TODO: study performance of UNIV_LIKELY branch prediction hints. */
762+ if (rw_lock_s_lock_low(lock, pass, file_name, line)) {
eccb488f
ER		 763
764 return; /* Success */
765 } else {
766 /* Did not succeed, try spin wait */
4b771f0e 767- mutex_exit(rw_lock_get_mutex(lock));
eccb488f
ER		 768
769 rw_lock_s_lock_spin(lock, pass, file_name, line);
770
4b771f0e
ER		 771@@ -259,86 +426,60 @@
772
773 /**********************************************************************
774 NOTE! Use the corresponding macro, not directly this function! Lock an
775-rw-lock in shared mode for the current thread if the lock can be acquired
776-immediately. */
777+rw-lock in exclusive mode for the current thread if the lock can be
778+obtained immediately. */
779 UNIV_INLINE
780 ibool
781-rw_lock_s_lock_func_nowait(
782+rw_lock_x_lock_func_nowait(
783 /*=======================*/
784 /* out: TRUE if success */
785 rw_lock_t* lock, /* in: pointer to rw-lock */
786 const char* file_name,/* in: file name where lock requested */
787 ulint line) /* in: line where requested */
eccb488f 788 {
4b771f0e
ER		 789- ibool success = FALSE;
790-
791- mutex_enter(rw_lock_get_mutex(lock));
792-
793- if (lock->writer == RW_LOCK_NOT_LOCKED) {
794- /* Set the shared lock by incrementing the reader count */
795- lock->reader_count++;
796+ os_thread_id_t curr_thread = os_thread_get_curr_id();
eccb488f 797
4b771f0e
ER		 798-#ifdef UNIV_SYNC_DEBUG
799- rw_lock_add_debug_info(lock, 0, RW_LOCK_SHARED, file_name,
800- line);
801-#endif
802+ ibool success;
803
804- lock->last_s_file_name = file_name;
805- lock->last_s_line = line;
eccb488f 806+#ifdef HAVE_ATOMIC_BUILTINS
4b771f0e 807+ success = os_compare_and_swap(&(lock->lock_word), X_LOCK_DECR, 0);
eccb488f 808+#else
4b771f0e
ER		 809
810+ success = FALSE;
811+ mutex_enter(&(lock->mutex));
812+ if (lock->lock_word == X_LOCK_DECR) {
813+ lock->lock_word = 0;
eccb488f
ER		 814 success = TRUE;
815 }
4b771f0e 816+ mutex_exit(&(lock->mutex));
eccb488f 817
4b771f0e
ER		 818- mutex_exit(rw_lock_get_mutex(lock));
819-
820- return(success);
821-}
eccb488f 822+#endif
4b771f0e
ER		 823+ if (success) {
824+ rw_lock_set_writer_id_and_recursion_flag(lock, TRUE);
825
826-/**********************************************************************
827-NOTE! Use the corresponding macro, not directly this function! Lock an
828-rw-lock in exclusive mode for the current thread if the lock can be
829-obtained immediately. */
830-UNIV_INLINE
831-ibool
832-rw_lock_x_lock_func_nowait(
833-/*=======================*/
834- /* out: TRUE if success */
835- rw_lock_t* lock, /* in: pointer to rw-lock */
836- const char* file_name,/* in: file name where lock requested */
837- ulint line) /* in: line where requested */
838-{
839- ibool success = FALSE;
840- os_thread_id_t curr_thread = os_thread_get_curr_id();
841- mutex_enter(rw_lock_get_mutex(lock));
842+ } else if (lock->recursive
843+ && os_thread_eq(lock->writer_thread, curr_thread)) {
844+ /* Relock: this lock_word modification is safe since no other
845+ threads can modify (lock, unlock, or reserve) lock_word while
846+ there is an exclusive writer and this is the writer thread. */
847+ lock->lock_word -= X_LOCK_DECR;
848
849- if (UNIV_UNLIKELY(rw_lock_get_reader_count(lock) != 0)) {
850- } else if (UNIV_LIKELY(rw_lock_get_writer(lock)
851- == RW_LOCK_NOT_LOCKED)) {
852- rw_lock_set_writer(lock, RW_LOCK_EX);
853- lock->writer_thread = curr_thread;
854- lock->pass = 0;
855- relock:
856- lock->writer_count++;
857+ ut_ad(((-lock->lock_word) % X_LOCK_DECR) == 0);
858
859+ } else {
860+ /* Failure */
861+ return(FALSE);
eccb488f 862+ }
4b771f0e
ER		 863 #ifdef UNIV_SYNC_DEBUG
864- rw_lock_add_debug_info(lock, 0, RW_LOCK_EX, file_name, line);
865+ rw_lock_add_debug_info(lock, 0, RW_LOCK_EX, file_name, line);
866 #endif
867
868- lock->last_x_file_name = file_name;
869- lock->last_x_line = line;
870-
871- success = TRUE;
872- } else if (rw_lock_get_writer(lock) == RW_LOCK_EX
873- && lock->pass == 0
874- && os_thread_eq(lock->writer_thread, curr_thread)) {
875- goto relock;
876- }
877-
878- mutex_exit(rw_lock_get_mutex(lock));
879+ lock->last_x_file_name = file_name;
880+ lock->last_x_line = line;
eccb488f 881
eccb488f
ER		 882 ut_ad(rw_lock_validate(lock));
883
4b771f0e
ER		 884- return(success);
885+ return(TRUE);
eccb488f
ER		 886 }
887
888 /**********************************************************************
4b771f0e 889@@ -354,39 +495,21 @@
eccb488f
ER		 890 #endif
891 )
892 {
4b771f0e 893- mutex_t* mutex = &(lock->mutex);
eccb488f 894- ibool sg = FALSE;
4b771f0e
ER		 895-
896- /* Acquire the mutex protecting the rw-lock fields */
897- mutex_enter(mutex);
898-
899- /* Reset the shared lock by decrementing the reader count */
900-
901- ut_a(lock->reader_count > 0);
902- lock->reader_count--;
903+ ut_ad((lock->lock_word % X_LOCK_DECR) != 0);
eccb488f
ER		 904
905 #ifdef UNIV_SYNC_DEBUG
906 rw_lock_remove_debug_info(lock, pass, RW_LOCK_SHARED);
4b771f0e
ER		 907 #endif
908
909- /* If there may be waiters and this was the last s-lock,
910- signal the object */
911+ /* Increment lock_word to indicate 1 less reader */
912+ if (rw_lock_lock_word_incr(lock, 1) == 0) {
eccb488f
ER		 913
914- if (UNIV_UNLIKELY(lock->waiters)
4b771f0e 915- && lock->reader_count == 0) {
eccb488f 916- sg = TRUE;
4b771f0e 917-
eccb488f 918- rw_lock_set_waiters(lock, 0);
4b771f0e
ER		 919- }
920-
921- mutex_exit(mutex);
922-
eccb488f
ER		 923- if (UNIV_UNLIKELY(sg)) {
924-#ifdef __WIN__
4b771f0e
ER		 925+ /* wait_ex waiter exists. It may not be asleep, but we signal
926+ anyway. We do not wake other waiters, because they can't
927+ exist without wait_ex waiter and wait_ex waiter goes first.*/
eccb488f
ER		 928 os_event_set(lock->wait_ex_event);
929-#endif
930- os_event_set(lock->event);
eccb488f 931 sync_array_object_signalled(sync_primary_wait_array);
4b771f0e 932+
eccb488f 933 }
45532174
ER		 934
935 ut_ad(rw_lock_validate(lock));
4b771f0e
ER		 936@@ -405,16 +528,15 @@
937 /*====================*/
938 rw_lock_t* lock) /* in: rw-lock */
939 {
940- /* Reset the shared lock by decrementing the reader count */
941-
942- ut_ad(lock->reader_count > 0);
943-
944- lock->reader_count--;
945+ ut_ad(lock->lock_word < X_LOCK_DECR);
eccb488f
ER		 946
947 #ifdef UNIV_SYNC_DEBUG
948 rw_lock_remove_debug_info(lock, 0, RW_LOCK_SHARED);
949 #endif
950
4b771f0e
ER		 951+ /* Decrease reader count by incrementing lock_word */
952+ lock->lock_word++;
953+
eccb488f 954 ut_ad(!lock->waiters);
eccb488f
ER		 955 ut_ad(rw_lock_validate(lock));
956 #ifdef UNIV_SYNC_PERF_STAT
4b771f0e 957@@ -435,42 +557,32 @@
eccb488f
ER		 958 #endif
959 )
960 {
961- ibool sg = FALSE;
4b771f0e
ER		 962-
963- /* Acquire the mutex protecting the rw-lock fields */
964- mutex_enter(&(lock->mutex));
965-
966- /* Reset the exclusive lock if this thread no longer has an x-mode
967- lock */
968-
969- ut_ad(lock->writer_count > 0);
970+ ut_ad((lock->lock_word % X_LOCK_DECR) == 0);
eccb488f 971
4b771f0e
ER		 972- lock->writer_count--;
973-
974- if (lock->writer_count == 0) {
975- rw_lock_set_writer(lock, RW_LOCK_NOT_LOCKED);
976+ /* lock->recursive flag also indicates if lock->writer_thread is
977+ valid or stale. If we are the last of the recursive callers
978+ then we must unset lock->recursive flag to indicate that the
979+ lock->writer_thread is now stale.
980+ Note that since we still hold the x-lock we can safely read the
981+ lock_word. */
982+ if (lock->lock_word == 0) {
983+ /* Last caller in a possible recursive chain. */
984+ lock->recursive = FALSE;
eccb488f 985 }
eccb488f
ER		 986
987 #ifdef UNIV_SYNC_DEBUG
988 rw_lock_remove_debug_info(lock, pass, RW_LOCK_EX);
989 #endif
990
4b771f0e 991- /* If there may be waiters, signal the lock */
eccb488f
ER		 992- if (UNIV_UNLIKELY(lock->waiters)
993- && lock->writer_count == 0) {
994-
995- sg = TRUE;
996- rw_lock_set_waiters(lock, 0);
4b771f0e
ER		 997- }
998-
999- mutex_exit(&(lock->mutex));
1000-
eccb488f 1001- if (UNIV_UNLIKELY(sg)) {
4b771f0e
ER		 1002-#ifdef __WIN__
1003- os_event_set(lock->wait_ex_event);
1004-#endif
eccb488f 1005- os_event_set(lock->event);
4b771f0e
ER		 1006- sync_array_object_signalled(sync_primary_wait_array);
1007+ if (rw_lock_lock_word_incr(lock, X_LOCK_DECR) == X_LOCK_DECR) {
1008+ /* Lock is now free. May have to signal read/write waiters.
1009+ We do not need to signal wait_ex waiters, since they cannot
1010+ exist when there is a writer. */
1011+ if (lock->waiters) {
1012+ rw_lock_reset_waiter_flag(lock);
1013+ os_event_set(lock->event);
1014+ sync_array_object_signalled(sync_primary_wait_array);
1015+ }
eccb488f
ER		 1016 }
1017
4b771f0e
ER		 1018 ut_ad(rw_lock_validate(lock));
1019@@ -492,18 +604,18 @@
1020 /* Reset the exclusive lock if this thread no longer has an x-mode
1021 lock */
eccb488f 1022
4b771f0e
ER		 1023- ut_ad(lock->writer_count > 0);
1024-
1025- lock->writer_count--;
1026-
1027- if (lock->writer_count == 0) {
1028- rw_lock_set_writer(lock, RW_LOCK_NOT_LOCKED);
1029- }
1030+ ut_ad((lock->lock_word % X_LOCK_DECR) == 0);
eccb488f 1031
4b771f0e 1032 #ifdef UNIV_SYNC_DEBUG
eccb488f
ER		 1033 rw_lock_remove_debug_info(lock, 0, RW_LOCK_EX);
1034 #endif
1035
4b771f0e
ER		 1036+ if (lock->lock_word == 0) {
1037+ lock->recursive = FALSE;
1038+ }
1039+
1040+ lock->lock_word += X_LOCK_DECR;
1041+
eccb488f 1042 ut_ad(!lock->waiters);
eccb488f
ER		 1043 ut_ad(rw_lock_validate(lock));
1044
4b771f0e
ER		 1045diff -ruN a/innobase/include/sync0sync.h b/innobase/include/sync0sync.h
1046--- a/innobase/include/sync0sync.h 2009-10-22 15:15:05.000000000 +0900
1047+++ b/innobase/include/sync0sync.h 2009-10-22 15:18:44.000000000 +0900
1048@@ -1,8 +1,31 @@
1049+/*****************************************************************************
1050+
1051+Copyright (c) 1995, 2009, Innobase Oy. All Rights Reserved.
1052+Copyright (c) 2008, Google Inc.
1053+
1054+Portions of this file contain modifications contributed and copyrighted by
1055+Google, Inc. Those modifications are gratefully acknowledged and are described
1056+briefly in the InnoDB documentation. The contributions by Google are
1057+incorporated with their permission, and subject to the conditions contained in
1058+the file COPYING.Google.
1059+
1060+This program is free software; you can redistribute it and/or modify it under
1061+the terms of the GNU General Public License as published by the Free Software
1062+Foundation; version 2 of the License.
1063+
1064+This program is distributed in the hope that it will be useful, but WITHOUT
1065+ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
1066+FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
1067+
1068+You should have received a copy of the GNU General Public License along with
1069+this program; if not, write to the Free Software Foundation, Inc., 59 Temple
1070+Place, Suite 330, Boston, MA 02111-1307 USA
1071+
1072+*****************************************************************************/
1073+
1074 /******************************************************
1075 Mutex, the basic synchronization primitive
1076
1077-(c) 1995 Innobase Oy
1078-
1079 Created 9/5/1995 Heikki Tuuri
1080 *******************************************************/
1081
1082@@ -465,8 +488,11 @@
1083 struct mutex_struct {
1084 os_event_t event; /* Used by sync0arr.c for the wait queue */
1085 ulint lock_word; /* This ulint is the target of the atomic
1086- test-and-set instruction in Win32 */
1087-#if !defined(_WIN32) || !defined(UNIV_CAN_USE_X86_ASSEMBLER)
1088+ test-and-set instruction in Win32 and
1089+ x86 32/64 with GCC 4.1.0 or later version */
1090+#if defined(_WIN32) && defined(UNIV_CAN_USE_X86_ASSEMBLER)
1091+#elif defined(HAVE_ATOMIC_BUILTINS)
1092+#else
1093 os_fast_mutex_t
1094 os_fast_mutex; /* In other systems we use this OS mutex
1095 in place of lock_word */
1096@@ -525,8 +551,7 @@
1097 /* The number of system calls made in this module. Intended for performance
1098 monitoring. */
1099
1100-extern ulint mutex_system_call_count;
1101-extern ulint mutex_exit_count;
1102+extern ib_longlong mutex_exit_count;
1103
1104 /* Latching order checks start when this is set TRUE */
1105 extern ibool sync_order_checks_on;
1106diff -ruN a/innobase/include/sync0sync.ic b/innobase/include/sync0sync.ic
1107--- a/innobase/include/sync0sync.ic 2009-09-10 04:02:59.000000000 +0900
1108+++ b/innobase/include/sync0sync.ic 2009-10-22 15:18:44.000000000 +0900
1109@@ -1,21 +1,34 @@
1110+/*****************************************************************************
1111+
1112+Copyright (c) 1995, 2009, Innobase Oy. All Rights Reserved.
1113+Copyright (c) 2008, Google Inc.
1114+
1115+Portions of this file contain modifications contributed and copyrighted by
1116+Google, Inc. Those modifications are gratefully acknowledged and are described
1117+briefly in the InnoDB documentation. The contributions by Google are
1118+incorporated with their permission, and subject to the conditions contained in
1119+the file COPYING.Google.
1120+
1121+This program is free software; you can redistribute it and/or modify it under
1122+the terms of the GNU General Public License as published by the Free Software
1123+Foundation; version 2 of the License.
1124+
1125+This program is distributed in the hope that it will be useful, but WITHOUT
1126+ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
1127+FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
1128+
1129+You should have received a copy of the GNU General Public License along with
1130+this program; if not, write to the Free Software Foundation, Inc., 59 Temple
1131+Place, Suite 330, Boston, MA 02111-1307 USA
1132+
1133+*****************************************************************************/
1134+
1135 /******************************************************
1136 Mutex, the basic synchronization primitive
1137
1138-(c) 1995 Innobase Oy
1139-
1140 Created 9/5/1995 Heikki Tuuri
1141 *******************************************************/
1142
1143-#if defined(not_defined) && defined(__GNUC__) && defined(UNIV_INTEL_X86)
1144-/* %z0: Use the size of operand %0 which in our case is *m to determine
1145-instruction size, it should end up as xchgl. "1" in the input constraint,
1146-says that "in" has to go in the same place as "out".*/
1147-#define TAS(m, in, out) \
1148- asm volatile ("xchg%z0 %2, %0" \
1149- : "=g" (*(m)), "=r" (out) \
1150- : "1" (in)) /* Note: "1" here refers to "=r" (out) */
1151-#endif
1152-
1153 /**********************************************************************
1154 Sets the waiters field in a mutex. */
1155
1156@@ -94,12 +107,8 @@
1157 /* mutex_fence(); */
1158
1159 return(res);
1160-#elif defined(not_defined) && defined(__GNUC__) && defined(UNIV_INTEL_X86)
1161- ulint res;
1162-
1163- TAS(&mutex->lock_word, 1, res);
1164-
1165- return(res);
1166+#elif defined(HAVE_ATOMIC_BUILTINS)
1167+ return __sync_lock_test_and_set(&(mutex->lock_word), 1);
1168 #else
1169 ibool ret;
1170
1171@@ -136,10 +145,11 @@
1172 __asm MOV EDX, 0
1173 __asm MOV ECX, lw
1174 __asm XCHG EDX, DWORD PTR [ECX]
1175-#elif defined(not_defined) && defined(__GNUC__) && defined(UNIV_INTEL_X86)
1176- ulint res;
1177-
1178- TAS(&mutex->lock_word, 0, res);
1179+#elif defined(HAVE_ATOMIC_BUILTINS)
1180+ /* In theory __sync_lock_release should be used to release the lock.
1181+ Unfortunately, it does not work properly alone. The workaround is
1182+ that more conservative __sync_lock_test_and_set is used instead. */
1183+ __sync_lock_test_and_set(&(mutex->lock_word), 0);
1184 #else
1185 mutex->lock_word = 0;
1186
1187diff -ruN a/innobase/row/row0sel.c b/innobase/row/row0sel.c
1188--- a/innobase/row/row0sel.c 2009-10-22 15:15:05.000000000 +0900
1189+++ b/innobase/row/row0sel.c 2009-10-22 15:18:44.000000000 +0900
1190@@ -1178,7 +1178,7 @@
1191 rw_lock_s_lock(&btr_search_latch);
1192
1193 search_latch_locked = TRUE;
1194- } else if (btr_search_latch.writer_is_wait_ex) {
1195+ } else if (rw_lock_get_writer(&btr_search_latch) == RW_LOCK_WAIT_EX) {
1196
1197 /* There is an x-latch request waiting: release the
1198 s-latch for a moment; as an s-latch here is often
1199@@ -3123,7 +3123,7 @@
1200 /* PHASE 0: Release a possible s-latch we are holding on the
1201 adaptive hash index latch if there is someone waiting behind */
1202
1203- if (UNIV_UNLIKELY(btr_search_latch.writer != RW_LOCK_NOT_LOCKED)
1204+ if (UNIV_UNLIKELY(rw_lock_get_writer(&btr_search_latch) != RW_LOCK_NOT_LOCKED)
1205 && trx->has_search_latch) {
1206
1207 /* There is an x-latch request on the adaptive hash index:
45532174 1208diff -ruN a/innobase/sync/sync0arr.c b/innobase/sync/sync0arr.c
4b771f0e
ER		 1209--- a/innobase/sync/sync0arr.c 2009-09-10 04:03:01.000000000 +0900
1210+++ b/innobase/sync/sync0arr.c 2009-10-22 15:18:44.000000000 +0900
1211@@ -1,8 +1,31 @@
1212+/*****************************************************************************
1213+
1214+Copyright (c) 1995, 2009, Innobase Oy. All Rights Reserved.
1215+Copyright (c) 2008, Google Inc.
1216+
1217+Portions of this file contain modifications contributed and copyrighted by
1218+Google, Inc. Those modifications are gratefully acknowledged and are described
1219+briefly in the InnoDB documentation. The contributions by Google are
1220+incorporated with their permission, and subject to the conditions contained in
1221+the file COPYING.Google.
1222+
1223+This program is free software; you can redistribute it and/or modify it under
1224+the terms of the GNU General Public License as published by the Free Software
1225+Foundation; version 2 of the License.
1226+
1227+This program is distributed in the hope that it will be useful, but WITHOUT
1228+ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
1229+FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
1230+
1231+You should have received a copy of the GNU General Public License along with
1232+this program; if not, write to the Free Software Foundation, Inc., 59 Temple
1233+Place, Suite 330, Boston, MA 02111-1307 USA
1234+
1235+*****************************************************************************/
1236+
1237 /******************************************************
1238 The wait array used in synchronization primitives
1239
1240-(c) 1995 Innobase Oy
1241-
1242 Created 9/5/1995 Heikki Tuuri
1243 *******************************************************/
1244
1245@@ -297,25 +320,21 @@
1246 }
1247
1248 /***********************************************************************
1249-Puts the cell event in reset state. */
1250+Returns the event that the thread owning the cell waits for. */
1251 static
1252-ib_longlong
1253-sync_cell_event_reset(
1254-/*==================*/
1255- /* out: value of signal_count
1256- at the time of reset. */
1257- ulint type, /* in: lock type mutex/rw_lock */
1258- void* object) /* in: the rw_lock/mutex object */
1259+os_event_t
1260+sync_cell_get_event(
1261+/*================*/
1262+ sync_cell_t* cell) /* in: non-empty sync array cell */
eccb488f 1263 {
4b771f0e
ER		 1264+ ulint type = cell->request_type;
1265+
eccb488f 1266 if (type == SYNC_MUTEX) {
4b771f0e 1267- return(os_event_reset(((mutex_t *) object)->event));
eccb488f 1268-#ifdef __WIN__
4b771f0e 1269+ return(((mutex_t *) cell->wait_object)->event);
eccb488f 1270 } else if (type == RW_LOCK_WAIT_EX) {
4b771f0e
ER		 1271- return(os_event_reset(
1272- ((rw_lock_t *) object)->wait_ex_event));
eccb488f
ER		 1273-#endif
1274- } else {
1275- return(os_event_reset(((rw_lock_t *) object)->event));
4b771f0e
ER		 1276+ return(((rw_lock_t *) cell->wait_object)->wait_ex_event);
1277+ } else { /* RW_LOCK_SHARED and RW_LOCK_EX wait on the same event */
1278+ return(((rw_lock_t *) cell->wait_object)->event);
eccb488f
ER		 1279 }
1280 }
1281
4b771f0e
ER		 1282@@ -334,6 +353,7 @@
1283 ulint* index) /* out: index of the reserved cell */
1284 {
1285 sync_cell_t* cell;
1286+ os_event_t event;
1287 ulint i;
1288
1289 ut_a(object);
1290@@ -372,8 +392,8 @@
1291 /* Make sure the event is reset and also store
1292 the value of signal_count at which the event
1293 was reset. */
1294- cell->signal_count = sync_cell_event_reset(type,
1295- object);
1296+ event = sync_cell_get_event(cell);
1297+ cell->signal_count = os_event_reset(event);
1298
1299 cell->reservation_time = time(NULL);
1300
1301@@ -413,19 +433,7 @@
1302 ut_a(!cell->waiting);
1303 ut_ad(os_thread_get_curr_id() == cell->thread);
1304
1305- if (cell->request_type == SYNC_MUTEX) {
1306- event = ((mutex_t*) cell->wait_object)->event;
eccb488f
ER		 1307-#ifdef __WIN__
1308- /* On windows if the thread about to wait is the one which
1309- has set the state of the rw_lock to RW_LOCK_WAIT_EX, then
1310- it waits on a special event i.e.: wait_ex_event. */
4b771f0e
ER		 1311- } else if (cell->request_type == RW_LOCK_WAIT_EX) {
1312- event = ((rw_lock_t*) cell->wait_object)->wait_ex_event;
eccb488f
ER		 1313-#endif
1314- } else {
1315- event = ((rw_lock_t*) cell->wait_object)->event;
4b771f0e
ER		 1316- }
1317-
1318+ event = sync_cell_get_event(cell);
eccb488f 1319 cell->waiting = TRUE;
4b771f0e
ER		 1320
1321 #ifdef UNIV_SYNC_DEBUG
1322@@ -464,6 +472,7 @@
eccb488f
ER		 1323 mutex_t* mutex;
1324 rw_lock_t* rwlock;
1325 ulint type;
1326+ ulint writer;
1327
1328 type = cell->request_type;
1329
4b771f0e 1330@@ -492,9 +501,7 @@
eccb488f
ER		 1331 (ulong) mutex->waiters);
1332
1333 } else if (type == RW_LOCK_EX
1334-#ifdef __WIN__
1335 || type == RW_LOCK_WAIT_EX
1336-#endif
1337 || type == RW_LOCK_SHARED) {
1338
4b771f0e
ER		 1339 fputs(type == RW_LOCK_EX ? "X-lock on" : "S-lock on", file);
1340@@ -505,21 +512,24 @@
eccb488f
ER		 1341 " RW-latch at %p created in file %s line %lu\n",
1342 rwlock, rwlock->cfile_name,
1343 (ulong) rwlock->cline);
1344- if (rwlock->writer != RW_LOCK_NOT_LOCKED) {
1345+ writer = rw_lock_get_writer(rwlock);
1346+ if (writer != RW_LOCK_NOT_LOCKED) {
1347 fprintf(file,
1348 "a writer (thread id %lu) has reserved it in mode %s",
1349 (ulong) os_thread_pf(rwlock->writer_thread),
1350- rwlock->writer == RW_LOCK_EX
1351+ writer == RW_LOCK_EX
1352 ? " exclusive\n"
1353 : " wait exclusive\n");
1354 }
1355
1356 fprintf(file,
1357- "number of readers %lu, waiters flag %lu\n"
4b771f0e
ER		 1358+ "number of readers %lu, waiters flag %lu, "
1359+ "lock_word: %lx\n"
eccb488f
ER		 1360 "Last time read locked in file %s line %lu\n"
1361 "Last time write locked in file %s line %lu\n",
4b771f0e
ER		 1362- (ulong) rwlock->reader_count,
1363+ (ulong) rw_lock_get_reader_count(rwlock),
1364 (ulong) rwlock->waiters,
1365+ rwlock->lock_word,
eccb488f
ER		 1366 rwlock->last_s_file_name,
1367 (ulong) rwlock->last_s_line,
1368 rwlock->last_x_file_name,
4b771f0e
ER		 1369@@ -773,28 +783,30 @@
1370 return(TRUE);
1371 }
1372
1373- } else if (cell->request_type == RW_LOCK_EX
1374- || cell->request_type == RW_LOCK_WAIT_EX) {
1375+ } else if (cell->request_type == RW_LOCK_EX) {
1376
1377 lock = cell->wait_object;
1378
1379- if (rw_lock_get_reader_count(lock) == 0
1380- && rw_lock_get_writer(lock) == RW_LOCK_NOT_LOCKED) {
1381+ if (lock->lock_word > 0) {
1382+ /* Either unlocked or only read locked. */
1383
1384 return(TRUE);
1385 }
1386
1387- if (rw_lock_get_reader_count(lock) == 0
1388- && rw_lock_get_writer(lock) == RW_LOCK_WAIT_EX
1389- && os_thread_eq(lock->writer_thread, cell->thread)) {
1390+ } else if (cell->request_type == RW_LOCK_WAIT_EX) {
1391+
1392+ lock = cell->wait_object;
1393+
1394+ /* lock_word == 0 means all readers have left */
1395+ if (lock->lock_word == 0) {
1396
1397 return(TRUE);
1398 }
1399-
1400 } else if (cell->request_type == RW_LOCK_SHARED) {
1401 lock = cell->wait_object;
1402
1403- if (rw_lock_get_writer(lock) == RW_LOCK_NOT_LOCKED) {
1404+ /* lock_word > 0 means no writer or reserved writer */
1405+ if (lock->lock_word > 0) {
1406
1407 return(TRUE);
1408 }
1409@@ -839,11 +851,15 @@
eccb488f
ER		 1410 /*========================*/
1411 sync_array_t* arr) /* in: wait array */
1412 {
1413+#ifdef HAVE_ATOMIC_BUILTINS
4b771f0e 1414+ (void) os_atomic_increment(&arr->sg_count, 1);
eccb488f
ER		 1415+#else
1416 sync_array_enter(arr);
1417
1418 arr->sg_count++;
1419
1420 sync_array_exit(arr);
1421+#endif
1422 }
1423
1424 /**************************************************************************
4b771f0e
ER		 1425@@ -859,6 +875,7 @@
1426 sync_cell_t* cell;
1427 ulint count;
1428 ulint i;
1429+ os_event_t event;
eccb488f 1430
4b771f0e
ER		 1431 sync_array_enter(arr);
1432
1433@@ -868,36 +885,20 @@
1434 while (count < arr->n_reserved) {
1435
1436 cell = sync_array_get_nth_cell(arr, i);
1437+ i++;
1438
1439- if (cell->wait_object != NULL) {
1440-
1441+ if (cell->wait_object == NULL) {
1442+ continue;
1443+ }
1444 count++;
eccb488f 1445
4b771f0e
ER		 1446 if (sync_arr_cell_can_wake_up(cell)) {
1447
1448- if (cell->request_type == SYNC_MUTEX) {
1449- mutex_t* mutex;
1450+ event = sync_cell_get_event(cell);
1451
1452- mutex = cell->wait_object;
1453- os_event_set(mutex->event);
1454-#ifdef __WIN__
1455- } else if (cell->request_type
1456- == RW_LOCK_WAIT_EX) {
1457- rw_lock_t* lock;
1458-
1459- lock = cell->wait_object;
1460- os_event_set(lock->wait_ex_event);
eccb488f
ER		 1461-#endif
1462- } else {
4b771f0e
ER		 1463- rw_lock_t* lock;
1464-
1465- lock = cell->wait_object;
eccb488f 1466- os_event_set(lock->event);
4b771f0e
ER		 1467- }
1468- }
1469+ os_event_set(event);
eccb488f 1470 }
4b771f0e
ER		 1471
1472- i++;
1473 }
1474
1475 sync_array_exit(arr);
1476@@ -1014,4 +1015,3 @@
1477
1478 sync_array_exit(arr);
1479 }
1480-
45532174 1481diff -ruN a/innobase/sync/sync0rw.c b/innobase/sync/sync0rw.c
4b771f0e
ER		 1482--- a/innobase/sync/sync0rw.c 2009-09-10 04:03:01.000000000 +0900
1483+++ b/innobase/sync/sync0rw.c 2009-10-22 15:18:44.000000000 +0900
1484@@ -1,8 +1,31 @@
1485+/*****************************************************************************
1486+
1487+Copyright (c) 1995, 2009, Innobase Oy. All Rights Reserved.
1488+Copyright (c) 2008, Google Inc.
1489+
1490+Portions of this file contain modifications contributed and copyrighted by
1491+Google, Inc. Those modifications are gratefully acknowledged and are described
1492+briefly in the InnoDB documentation. The contributions by Google are
1493+incorporated with their permission, and subject to the conditions contained in
1494+the file COPYING.Google.
1495+
1496+This program is free software; you can redistribute it and/or modify it under
1497+the terms of the GNU General Public License as published by the Free Software
1498+Foundation; version 2 of the License.
1499+
1500+This program is distributed in the hope that it will be useful, but WITHOUT
1501+ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
1502+FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
1503+
1504+You should have received a copy of the GNU General Public License along with
1505+this program; if not, write to the Free Software Foundation, Inc., 59 Temple
1506+Place, Suite 330, Boston, MA 02111-1307 USA
1507+
1508+*****************************************************************************/
1509+
1510 /******************************************************
1511 The read-write lock (for thread synchronization)
1512
1513-(c) 1995 Innobase Oy
1514-
1515 Created 9/11/1995 Heikki Tuuri
1516 *******************************************************/
1517
1518@@ -15,17 +38,110 @@
1519 #include "mem0mem.h"
1520 #include "srv0srv.h"
1521
1522-ulint rw_s_system_call_count = 0;
1523-ulint rw_s_spin_wait_count = 0;
1524-ulint rw_s_os_wait_count = 0;
1525+/*
1526+ IMPLEMENTATION OF THE RW_LOCK
1527+ =============================
1528+The status of a rw_lock is held in lock_word. The initial value of lock_word is
1529+X_LOCK_DECR. lock_word is decremented by 1 for each s-lock and by X_LOCK_DECR
1530+for each x-lock. This describes the lock state for each value of lock_word:
1531+
1532+lock_word == X_LOCK_DECR: Unlocked.
1533+0 < lock_word < X_LOCK_DECR: Read locked, no waiting writers.
1534+ (X_LOCK_DECR - lock_word) is the
1535+ number of readers that hold the lock.
1536+lock_word == 0: Write locked
1537+-X_LOCK_DECR < lock_word < 0: Read locked, with a waiting writer.
1538+ (-lock_word) is the number of readers
1539+ that hold the lock.
1540+lock_word <= -X_LOCK_DECR: Recursively write locked. lock_word has been
1541+ decremented by X_LOCK_DECR once for each lock,
1542+ so the number of locks is:
1543+ ((-lock_word) / X_LOCK_DECR) + 1
1544+When lock_word <= -X_LOCK_DECR, we also know that lock_word % X_LOCK_DECR == 0:
1545+other values of lock_word are invalid.
1546+
1547+The lock_word is always read and updated atomically and consistently, so that
1548+it always represents the state of the lock, and the state of the lock changes
1549+with a single atomic operation. This lock_word holds all of the information
1550+that a thread needs in order to determine if it is eligible to gain the lock
1551+or if it must spin or sleep. The one exception to this is that writer_thread
1552+must be verified before recursive write locks: to solve this scenario, we make
1553+writer_thread readable by all threads, but only writeable by the x-lock holder.
1554+
1555+The other members of the lock obey the following rules to remain consistent:
1556+
1557+recursive: This and the writer_thread field together control the
1558+ behaviour of recursive x-locking.
1559+ lock->recursive must be FALSE in following states:
1560+ 1) The writer_thread contains garbage i.e.: the
1561+ lock has just been initialized.
1562+ 2) The lock is not x-held and there is no
1563+ x-waiter waiting on WAIT_EX event.
1564+ 3) The lock is x-held or there is an x-waiter
1565+ waiting on WAIT_EX event but the 'pass' value
1566+ is non-zero.
1567+ lock->recursive is TRUE iff:
1568+ 1) The lock is x-held or there is an x-waiter
1569+ waiting on WAIT_EX event and the 'pass' value
1570+ is zero.
1571+ This flag must be set after the writer_thread field
1572+ has been updated with a memory ordering barrier.
1573+ It is unset before the lock_word has been incremented.
1574+writer_thread: Is used only in recursive x-locking. Can only be safely
1575+ read iff lock->recursive flag is TRUE.
1576+ This field is uninitialized at lock creation time and
1577+ is updated atomically when x-lock is acquired or when
1578+ move_ownership is called. A thread is only allowed to
1579+ set the value of this field to it's thread_id i.e.: a
1580+ thread cannot set writer_thread to some other thread's
1581+ id.
1582+waiters: May be set to 1 anytime, but to avoid unnecessary wake-up
1583+ signals, it should only be set to 1 when there are threads
1584+ waiting on event. Must be 1 when a writer starts waiting to
1585+ ensure the current x-locking thread sends a wake-up signal
1586+ during unlock. May only be reset to 0 immediately before a
1587+ a wake-up signal is sent to event. On most platforms, a
1588+ memory barrier is required after waiters is set, and before
1589+ verifying lock_word is still held, to ensure some unlocker
1590+ really does see the flags new value.
1591+event: Threads wait on event for read or writer lock when another
1592+ thread has an x-lock or an x-lock reservation (wait_ex). A
1593+ thread may only wait on event after performing the following
1594+ actions in order:
1595+ (1) Record the counter value of event (with os_event_reset).
1596+ (2) Set waiters to 1.
1597+ (3) Verify lock_word <= 0.
1598+ (1) must come before (2) to ensure signal is not missed.
1599+ (2) must come before (3) to ensure a signal is sent.
1600+ These restrictions force the above ordering.
1601+ Immediately before sending the wake-up signal, we should:
1602+ (1) Verify lock_word == X_LOCK_DECR (unlocked)
1603+ (2) Reset waiters to 0.
1604+wait_ex_event: A thread may only wait on the wait_ex_event after it has
1605+ performed the following actions in order:
1606+ (1) Decrement lock_word by X_LOCK_DECR.
1607+ (2) Record counter value of wait_ex_event (os_event_reset,
1608+ called from sync_array_reserve_cell).
1609+ (3) Verify that lock_word < 0.
1610+ (1) must come first to ensures no other threads become reader
1611+ or next writer, and notifies unlocker that signal must be sent.
1612+ (2) must come before (3) to ensure the signal is not missed.
1613+ These restrictions force the above ordering.
1614+ Immediately before sending the wake-up signal, we should:
1615+ Verify lock_word == 0 (waiting thread holds x_lock)
1616+*/
1617+
1618+ib_longlong rw_s_spin_wait_count = 0;
1619+ib_longlong rw_s_spin_round_count = 0;
1620+ib_longlong rw_s_os_wait_count = 0;
1621+
1622+ib_longlong rw_s_exit_count = 0;
1623+
1624+ib_longlong rw_x_spin_wait_count = 0;
1625+ib_longlong rw_x_spin_round_count = 0;
1626+ib_longlong rw_x_os_wait_count = 0;
1627
1628-ulint rw_s_exit_count = 0;
1629-
1630-ulint rw_x_system_call_count = 0;
1631-ulint rw_x_spin_wait_count = 0;
1632-ulint rw_x_os_wait_count = 0;
1633-
1634-ulint rw_x_exit_count = 0;
1635+ib_longlong rw_x_exit_count = 0;
1636
1637 /* The global list of rw-locks */
1638 rw_lock_list_t rw_lock_list;
1639@@ -99,22 +215,30 @@
eccb488f
ER		 1640 object is created, then the following call initializes
1641 the sync system. */
1642
1643+#ifndef HAVE_ATOMIC_BUILTINS
1644 mutex_create(rw_lock_get_mutex(lock));
1645 mutex_set_level(rw_lock_get_mutex(lock), SYNC_NO_ORDER_CHECK);
1646
4b771f0e
ER		 1647 lock->mutex.cfile_name = cfile_name;
1648 lock->mutex.cline = cline;
1649-#if defined UNIV_DEBUG && !defined UNIV_HOTBACKUP
1650+# if defined UNIV_DEBUG && !defined UNIV_HOTBACKUP
eccb488f
ER		 1651 lock->mutex.cmutex_name = cmutex_name;
1652 lock->mutex.mutex_type = 1;
4b771f0e
ER		 1653-#endif /* UNIV_DEBUG && !UNIV_HOTBACKUP */
1654+# endif /* UNIV_DEBUG && !UNIV_HOTBACKUP */
eccb488f
ER		 1655
1656- rw_lock_set_waiters(lock, 0);
4b771f0e
ER		 1657- rw_lock_set_writer(lock, RW_LOCK_NOT_LOCKED);
1658- lock->writer_count = 0;
1659- rw_lock_set_reader_count(lock, 0);
1660-
1661- lock->writer_is_wait_ex = FALSE;
1662+#else /* HAVE_ATOMIC_BUILTINS */
1663+# ifdef UNIV_DEBUG
1664+ UT_NOT_USED(cmutex_name);
1665+# endif
1666+#endif /* HAVE_ATOMIC_BUILTINS */
1667+
1668+ lock->lock_word = X_LOCK_DECR;
1669+ lock->waiters = 0;
1670+
1671+ /* We set this value to signify that lock->writer_thread
1672+ contains garbage at initialization and cannot be used for
1673+ recursive x-locking. */
1674+ lock->recursive = FALSE;
1675
1676 #ifdef UNIV_SYNC_DEBUG
1677 UT_LIST_INIT(lock->debug_list);
1678@@ -126,15 +250,13 @@
1679 lock->cfile_name = cfile_name;
1680 lock->cline = cline;
1681
1682+ lock->count_os_wait = 0;
1683 lock->last_s_file_name = "not yet reserved";
eccb488f
ER		 1684 lock->last_x_file_name = "not yet reserved";
1685 lock->last_s_line = 0;
1686 lock->last_x_line = 0;
4b771f0e 1687 lock->event = os_event_create(NULL);
eccb488f
ER		 1688-
1689-#ifdef __WIN__
eccb488f
ER		 1690 lock->wait_ex_event = os_event_create(NULL);
1691-#endif
1692
1693 mutex_enter(&rw_lock_list_mutex);
1694
4b771f0e
ER		 1695@@ -158,23 +280,17 @@
1696 /*=========*/
1697 rw_lock_t* lock) /* in: rw-lock */
1698 {
1699-#ifdef UNIV_DEBUG
eccb488f 1700 ut_a(rw_lock_validate(lock));
4b771f0e
ER		 1701-#endif /* UNIV_DEBUG */
1702- ut_a(rw_lock_get_writer(lock) == RW_LOCK_NOT_LOCKED);
eccb488f 1703- ut_a(rw_lock_get_waiters(lock) == 0);
4b771f0e
ER		 1704- ut_a(rw_lock_get_reader_count(lock) == 0);
1705+ ut_a(lock->lock_word == X_LOCK_DECR);
eccb488f 1706
4b771f0e
ER		 1707- lock->magic_n = 0;
1708-
eccb488f
ER		 1709+#ifndef HAVE_ATOMIC_BUILTINS
1710 mutex_free(rw_lock_get_mutex(lock));
4b771f0e 1711+#endif /* HAVE_ATOMIC_BUILTINS */
eccb488f
ER		 1712
1713 mutex_enter(&rw_lock_list_mutex);
4b771f0e
ER		 1714 os_event_free(lock->event);
1715
eccb488f 1716-#ifdef __WIN__
eccb488f
ER		 1717 os_event_free(lock->wait_ex_event);
1718-#endif
1719
1720 if (UT_LIST_GET_PREV(list, lock)) {
1721 ut_a(UT_LIST_GET_PREV(list, lock)->magic_n == RW_LOCK_MAGIC_N);
4b771f0e
ER		 1722@@ -186,6 +302,8 @@
1723 UT_LIST_REMOVE(list, rw_lock_list, lock);
eccb488f 1724
4b771f0e 1725 mutex_exit(&rw_lock_list_mutex);
eccb488f 1726+
4b771f0e
ER		 1727+ lock->magic_n = 0;
1728 }
1729
1730 /**********************************************************************
1731@@ -199,19 +317,12 @@
eccb488f
ER		 1732 {
1733 ut_a(lock);
1734
4b771f0e
ER		 1735- mutex_enter(rw_lock_get_mutex(lock));
1736+ ulint waiters = rw_lock_get_waiters(lock);
1737+ lint lock_word = lock->lock_word;
eccb488f
ER		 1738
1739 ut_a(lock->magic_n == RW_LOCK_MAGIC_N);
4b771f0e
ER		 1740- ut_a((rw_lock_get_reader_count(lock) == 0)
1741- || (rw_lock_get_writer(lock) != RW_LOCK_EX));
45532174
ER		 1742- ut_a((rw_lock_get_writer(lock) == RW_LOCK_EX)
1743- || (rw_lock_get_writer(lock) == RW_LOCK_WAIT_EX)
1744- || (rw_lock_get_writer(lock) == RW_LOCK_NOT_LOCKED));
eccb488f
ER		 1745- ut_a((rw_lock_get_waiters(lock) == 0)
1746- || (rw_lock_get_waiters(lock) == 1));
4b771f0e
ER		 1747- ut_a((lock->writer != RW_LOCK_EX) || (lock->writer_count > 0));
1748-
1749- mutex_exit(rw_lock_get_mutex(lock));
1750+ ut_a(waiters == 0 || waiters == 1);
1751+ ut_a(lock_word > -X_LOCK_DECR ||(-lock_word) % X_LOCK_DECR == 0);
eccb488f
ER		 1752
1753 return(TRUE);
1754 }
4b771f0e
ER		 1755@@ -232,18 +343,15 @@
1756 ulint line) /* in: line where requested */
1757 {
1758 ulint index; /* index of the reserved wait cell */
1759- ulint i; /* spin round count */
1760+ ulint i = 0; /* spin round count */
1761
eccb488f
ER		 1762 ut_ad(rw_lock_validate(lock));
1763
4b771f0e 1764+ rw_s_spin_wait_count++; /* Count calls to this function */
eccb488f 1765 lock_loop:
4b771f0e 1766- rw_s_spin_wait_count++;
eccb488f
ER		 1767
1768 /* Spin waiting for the writer field to become free */
1769- i = 0;
4b771f0e 1770-
eccb488f
ER		 1771- while (rw_lock_get_writer(lock) != RW_LOCK_NOT_LOCKED
1772- && i < SYNC_SPIN_ROUNDS) {
4b771f0e 1773+ while (i < SYNC_SPIN_ROUNDS && lock->lock_word <= 0) {
eccb488f
ER		 1774 if (srv_spin_wait_delay) {
1775 ut_delay(ut_rnd_interval(0, srv_spin_wait_delay));
1776 }
4b771f0e 1777@@ -262,28 +370,32 @@
eccb488f
ER		 1778 lock->cfile_name, (ulong) lock->cline, (ulong) i);
1779 }
1780
4b771f0e
ER		 1781- mutex_enter(rw_lock_get_mutex(lock));
1782-
eccb488f 1783 /* We try once again to obtain the lock */
4b771f0e 1784-
eccb488f 1785 if (TRUE == rw_lock_s_lock_low(lock, pass, file_name, line)) {
4b771f0e
ER		 1786- mutex_exit(rw_lock_get_mutex(lock));
1787+ rw_s_spin_round_count += i;
eccb488f
ER		 1788
1789 return; /* Success */
1790 } else {
4b771f0e
ER		 1791- /* If we get here, locking did not succeed, we may
1792- suspend the thread to wait in the wait array */
1793
1794- rw_s_system_call_count++;
eccb488f 1795+ if (i < SYNC_SPIN_ROUNDS) {
4b771f0e 1796+ goto lock_loop;
eccb488f 1797+ }
4b771f0e
ER		 1798+
1799+ rw_s_spin_round_count += i;
eccb488f 1800
4b771f0e
ER		 1801 sync_array_reserve_cell(sync_primary_wait_array,
1802 lock, RW_LOCK_SHARED,
eccb488f
ER		 1803 file_name, line,
1804 &index);
1805
1806- rw_lock_set_waiters(lock, 1);
4b771f0e
ER		 1807-
1808- mutex_exit(rw_lock_get_mutex(lock));
1809+ /* Set waiters before checking lock_word to ensure wake-up
1810+ signal is sent. This may lead to some unnecessary signals. */
1811+ rw_lock_set_waiter_flag(lock);
45532174 1812+
4b771f0e
ER		 1813+ if (TRUE == rw_lock_s_lock_low(lock, pass, file_name, line)) {
1814+ sync_array_free_cell(sync_primary_wait_array, index);
1815+ return; /* Success */
337bc045 1816+ }
eccb488f
ER		 1817
1818 if (srv_print_latch_waits) {
1819 fprintf(stderr,
4b771f0e
ER		 1820@@ -292,11 +404,13 @@
1821 lock, lock->cfile_name, (ulong) lock->cline);
1822 }
eccb488f 1823
4b771f0e
ER		 1824- rw_s_system_call_count++;
1825+ /* these stats may not be accurate */
1826+ lock->count_os_wait++;
1827 rw_s_os_wait_count++;
eccb488f 1828
4b771f0e 1829 sync_array_wait_event(sync_primary_wait_array, index);
eccb488f 1830
4b771f0e
ER		 1831+ i = 0;
1832 goto lock_loop;
1833 }
1834 }
1835@@ -318,114 +432,130 @@
1836 {
1837 ut_ad(rw_lock_is_locked(lock, RW_LOCK_EX));
eccb488f 1838
4b771f0e
ER		 1839- mutex_enter(&(lock->mutex));
1840-
1841- lock->writer_thread = os_thread_get_curr_id();
1842-
1843- lock->pass = 0;
1844-
1845- mutex_exit(&(lock->mutex));
1846+ rw_lock_set_writer_id_and_recursion_flag(lock, TRUE);
eccb488f
ER		 1847 }
1848
1849 /**********************************************************************
4b771f0e
ER		 1850-Low-level function for acquiring an exclusive lock. */
1851+Function for the next writer to call. Waits for readers to exit.
1852+The caller must have already decremented lock_word by X_LOCK_DECR.*/
1853 UNIV_INLINE
1854-ulint
1855-rw_lock_x_lock_low(
1856-/*===============*/
1857- /* out: RW_LOCK_NOT_LOCKED if did
1858- not succeed, RW_LOCK_EX if success,
1859- RW_LOCK_WAIT_EX, if got wait reservation */
1860+void
1861+rw_lock_x_lock_wait(
1862+/*================*/
1863 rw_lock_t* lock, /* in: pointer to rw-lock */
1864+#ifdef UNIV_SYNC_DEBUG
1865 ulint pass, /* in: pass value; != 0, if the lock will
1866 be passed to another thread to unlock */
1867+#endif
eccb488f
ER		 1868 const char* file_name,/* in: file name where lock requested */
1869 ulint line) /* in: line where requested */
1870 {
4b771f0e
ER		 1871-#ifdef UNIV_SYNC_DEBUG
1872- ut_ad(mutex_own(rw_lock_get_mutex(lock)));
1873-#endif /* UNIV_SYNC_DEBUG */
1874- if (rw_lock_get_writer(lock) == RW_LOCK_NOT_LOCKED) {
1875+ ulint index;
1876+ ulint i = 0;
1877
1878- if (rw_lock_get_reader_count(lock) == 0) {
1879+ ut_ad(lock->lock_word <= 0);
eccb488f 1880+
4b771f0e
ER		 1881+ while (lock->lock_word < 0) {
1882+ if (srv_spin_wait_delay) {
1883+ ut_delay(ut_rnd_interval(0, srv_spin_wait_delay));
eccb488f 1884+ }
4b771f0e
ER		 1885+ if(i < SYNC_SPIN_ROUNDS) {
1886+ i++;
1887+ continue;
eccb488f 1888+ }
4b771f0e
ER		 1889
1890- rw_lock_set_writer(lock, RW_LOCK_EX);
1891- lock->writer_thread = os_thread_get_curr_id();
1892- lock->writer_count++;
1893- lock->pass = pass;
1894+ /* If there is still a reader, then go to sleep.*/
1895+ rw_x_spin_round_count += i;
1896+ i = 0;
1897+ sync_array_reserve_cell(sync_primary_wait_array,
1898+ lock,
1899+ RW_LOCK_WAIT_EX,
1900+ file_name, line,
1901+ &index);
1902+ /* Check lock_word to ensure wake-up isn't missed.*/
1903+ if(lock->lock_word < 0) {
1904
1905+ /* these stats may not be accurate */
1906+ lock->count_os_wait++;
1907+ rw_x_os_wait_count++;
eccb488f 1908+
4b771f0e
ER		 1909+ /* Add debug info as it is needed to detect possible
1910+ deadlock. We must add info for WAIT_EX thread for
1911+ deadlock detection to work properly. */
1912 #ifdef UNIV_SYNC_DEBUG
1913- rw_lock_add_debug_info(lock, pass, RW_LOCK_EX,
1914+ rw_lock_add_debug_info(lock, pass, RW_LOCK_WAIT_EX,
1915 file_name, line);
1916 #endif
1917- lock->last_x_file_name = file_name;
1918- lock->last_x_line = line;
1919-
1920- /* Locking succeeded, we may return */
1921- return(RW_LOCK_EX);
1922- } else {
1923- /* There are readers, we have to wait */
1924- rw_lock_set_writer(lock, RW_LOCK_WAIT_EX);
1925- lock->writer_thread = os_thread_get_curr_id();
1926- lock->pass = pass;
1927- lock->writer_is_wait_ex = TRUE;
1928
1929+ sync_array_wait_event(sync_primary_wait_array,
1930+ index);
1931 #ifdef UNIV_SYNC_DEBUG
1932- rw_lock_add_debug_info(lock, pass, RW_LOCK_WAIT_EX,
1933- file_name, line);
1934+ rw_lock_remove_debug_info(lock, pass,
1935+ RW_LOCK_WAIT_EX);
1936 #endif
1937-
1938- return(RW_LOCK_WAIT_EX);
1939+ /* It is possible to wake when lock_word < 0.
1940+ We must pass the while-loop check to proceed.*/
1941+ } else {
1942+ sync_array_free_cell(sync_primary_wait_array,
1943+ index);
1944 }
eccb488f 1945+ }
4b771f0e
ER		 1946+ rw_x_spin_round_count += i;
1947+}
1948
1949- } else if ((rw_lock_get_writer(lock) == RW_LOCK_WAIT_EX)
1950- && os_thread_eq(lock->writer_thread,
1951- os_thread_get_curr_id())) {
1952+/**********************************************************************
1953+Low-level function for acquiring an exclusive lock. */
1954+UNIV_INLINE
1955+ibool
1956+rw_lock_x_lock_low(
1957+/*===============*/
1958+ /* out: RW_LOCK_NOT_LOCKED if did
1959+ not succeed, RW_LOCK_EX if success. */
1960+ rw_lock_t* lock, /* in: pointer to rw-lock */
1961+ ulint pass, /* in: pass value; != 0, if the lock will
1962+ be passed to another thread to unlock */
1963+ const char* file_name,/* in: file name where lock requested */
1964+ ulint line) /* in: line where requested */
1965+{
1966+ os_thread_id_t curr_thread = os_thread_get_curr_id();
1967
1968- if (rw_lock_get_reader_count(lock) == 0) {
1969+ if (rw_lock_lock_word_decr(lock, X_LOCK_DECR)) {
1970
1971- rw_lock_set_writer(lock, RW_LOCK_EX);
1972- lock->writer_count++;
1973- lock->pass = pass;
1974- lock->writer_is_wait_ex = FALSE;
1975+ /* lock->recursive also tells us if the writer_thread
1976+ field is stale or active. As we are going to write
1977+ our own thread id in that field it must be that the
1978+ current writer_thread value is not active. */
1979+ ut_a(!lock->recursive);
1980
1981+ /* Decrement occurred: we are writer or next-writer. */
1982+ rw_lock_set_writer_id_and_recursion_flag(lock,
1983+ pass ? FALSE : TRUE);
eccb488f 1984+
4b771f0e 1985+ rw_lock_x_lock_wait(lock,
eccb488f 1986 #ifdef UNIV_SYNC_DEBUG
4b771f0e
ER		 1987- rw_lock_remove_debug_info(lock, pass, RW_LOCK_WAIT_EX);
1988- rw_lock_add_debug_info(lock, pass, RW_LOCK_EX,
1989- file_name, line);
1990+ pass,
1991 #endif
1992+ file_name, line);
1993
1994- lock->last_x_file_name = file_name;
1995- lock->last_x_line = line;
1996-
1997- /* Locking succeeded, we may return */
1998- return(RW_LOCK_EX);
1999+ } else {
2000+ /* Decrement failed: relock or failed lock */
2001+ if (!pass && lock->recursive
2002+ && os_thread_eq(lock->writer_thread, curr_thread)) {
2003+ /* Relock */
2004+ lock->lock_word -= X_LOCK_DECR;
2005+ } else {
2006+ /* Another thread locked before us */
2007+ return(FALSE);
2008 }
2009-
2010- return(RW_LOCK_WAIT_EX);
2011-
2012- } else if ((rw_lock_get_writer(lock) == RW_LOCK_EX)
2013- && os_thread_eq(lock->writer_thread,
2014- os_thread_get_curr_id())
2015- && (lock->pass == 0)
2016- && (pass == 0)) {
2017-
2018- lock->writer_count++;
2019-
2020+ }
2021 #ifdef UNIV_SYNC_DEBUG
2022- rw_lock_add_debug_info(lock, pass, RW_LOCK_EX, file_name,
2023- line);
2024+ rw_lock_add_debug_info(lock, pass, RW_LOCK_EX,
2025+ file_name, line);
2026 #endif
2027+ lock->last_x_file_name = file_name;
2028+ lock->last_x_line = (unsigned int) line;
2029
2030- lock->last_x_file_name = file_name;
2031- lock->last_x_line = line;
2032-
2033- /* Locking succeeded, we may return */
2034- return(RW_LOCK_EX);
2035- }
2036-
2037- /* Locking did not succeed */
2038- return(RW_LOCK_NOT_LOCKED);
2039+ return(TRUE);
2040 }
eccb488f 2041
4b771f0e
ER		 2042 /**********************************************************************
2043@@ -448,47 +578,30 @@
eccb488f
ER		 2044 ulint line) /* in: line where requested */
2045 {
2046 ulint index; /* index of the reserved wait cell */
2047- ulint state; /* lock state acquired */
eccb488f 2048 ulint i; /* spin round count */
4b771f0e 2049+ ibool spinning = FALSE;
eccb488f
ER		 2050
2051 ut_ad(rw_lock_validate(lock));
2052
4b771f0e
ER		 2053-lock_loop:
2054- /* Acquire the mutex protecting the rw-lock fields */
2055- mutex_enter_fast(&(lock->mutex));
2056-
2057- state = rw_lock_x_lock_low(lock, pass, file_name, line);
eccb488f 2058+ i = 0;
eccb488f 2059
4b771f0e
ER		 2060- mutex_exit(&(lock->mutex));
2061+lock_loop:
eccb488f 2062
4b771f0e
ER		 2063- if (state == RW_LOCK_EX) {
2064+ if (rw_lock_x_lock_low(lock, pass, file_name, line)) {
2065+ rw_x_spin_round_count += i;
eccb488f
ER		 2066
2067 return; /* Locking succeeded */
eccb488f 2068
4b771f0e
ER		 2069- } else if (state == RW_LOCK_NOT_LOCKED) {
2070-
2071- /* Spin waiting for the writer field to become free */
eccb488f 2072- i = 0;
4b771f0e 2073-
eccb488f
ER		 2074- while (rw_lock_get_writer(lock) != RW_LOCK_NOT_LOCKED
2075- && i < SYNC_SPIN_ROUNDS) {
4b771f0e
ER		 2076- if (srv_spin_wait_delay) {
2077- ut_delay(ut_rnd_interval(0,
2078- srv_spin_wait_delay));
2079- }
2080+ } else {
2081
2082- i++;
2083- }
2084- if (i == SYNC_SPIN_ROUNDS) {
2085- os_thread_yield();
2086+ if (!spinning) {
2087+ spinning = TRUE;
2088+ rw_x_spin_wait_count++;
2089 }
2090- } else if (state == RW_LOCK_WAIT_EX) {
eccb488f 2091
4b771f0e 2092- /* Spin waiting for the reader count field to become zero */
eccb488f 2093- i = 0;
4b771f0e
ER		 2094-
2095- while (rw_lock_get_reader_count(lock) != 0
2096- && i < SYNC_SPIN_ROUNDS) {
2097+ /* Spin waiting for the lock_word to become free */
2098+ while (i < SYNC_SPIN_ROUNDS
2099+ && lock->lock_word <= 0) {
eccb488f
ER		 2100 if (srv_spin_wait_delay) {
2101 ut_delay(ut_rnd_interval(0,
4b771f0e
ER		 2102 srv_spin_wait_delay));
2103@@ -498,12 +611,13 @@
2104 }
2105 if (i == SYNC_SPIN_ROUNDS) {
eccb488f 2106 os_thread_yield();
4b771f0e
ER		 2107+ } else {
2108+ goto lock_loop;
eccb488f 2109 }
4b771f0e 2110- } else {
eccb488f 2111- i = 0; /* Eliminate a compiler warning */
4b771f0e 2112- ut_error;
eccb488f
ER		 2113 }
2114
4b771f0e 2115+ rw_x_spin_round_count += i;
eccb488f 2116+
4b771f0e
ER		 2117 if (srv_print_latch_waits) {
2118 fprintf(stderr,
2119 "Thread %lu spin wait rw-x-lock at %p cfile %s cline %lu rnds %lu\n",
2120@@ -511,39 +625,20 @@
2121 lock->cfile_name, (ulong) lock->cline, (ulong) i);
eccb488f 2122 }
45532174 2123
4b771f0e
ER		 2124- rw_x_spin_wait_count++;
2125-
2126- /* We try once again to obtain the lock. Acquire the mutex protecting
2127- the rw-lock fields */
2128-
2129- mutex_enter(rw_lock_get_mutex(lock));
2130-
2131- state = rw_lock_x_lock_low(lock, pass, file_name, line);
2132-
2133- if (state == RW_LOCK_EX) {
2134- mutex_exit(rw_lock_get_mutex(lock));
2135-
2136- return; /* Locking succeeded */
2137- }
2138-
2139- rw_x_system_call_count++;
2140-
eccb488f
ER		 2141 sync_array_reserve_cell(sync_primary_wait_array,
2142 lock,
2143-#ifdef __WIN__
2144- /* On windows RW_LOCK_WAIT_EX signifies
2145- that this thread should wait on the
2146- special wait_ex_event. */
4b771f0e
ER		 2147- (state == RW_LOCK_WAIT_EX)
2148- ? RW_LOCK_WAIT_EX :
eccb488f
ER		 2149-#endif
2150 RW_LOCK_EX,
2151 file_name, line,
2152 &index);
2153
2154- rw_lock_set_waiters(lock, 1);
4b771f0e
ER		 2155-
2156- mutex_exit(rw_lock_get_mutex(lock));
2157+ /* Waiters must be set before checking lock_word, to ensure signal
2158+ is sent. This could lead to a few unnecessary wake-up signals. */
2159+ rw_lock_set_waiter_flag(lock);
2160+
2161+ if (rw_lock_x_lock_low(lock, pass, file_name, line)) {
2162+ sync_array_free_cell(sync_primary_wait_array, index);
2163+ return; /* Locking succeeded */
eccb488f 2164+ }
eccb488f
ER		 2165
2166 if (srv_print_latch_waits) {
2167 fprintf(stderr,
4b771f0e
ER		 2168@@ -552,11 +647,13 @@
2169 lock->cfile_name, (ulong) lock->cline);
2170 }
2171
2172- rw_x_system_call_count++;
2173+ /* these stats may not be accurate */
2174+ lock->count_os_wait++;
2175 rw_x_os_wait_count++;
2176
2177 sync_array_wait_event(sync_primary_wait_array, index);
2178
2179+ i = 0;
2180 goto lock_loop;
2181 }
2182
2183@@ -697,7 +794,9 @@
2184 rw_lock_t* lock, /* in: rw-lock */
2185 ulint level) /* in: level */
2186 {
2187+#ifdef UNIV_SYNC_DEBUG
2188 lock->level = level;
2189+#endif /* UNIV_SYNC_DEBUG */
2190 }
2191
2192 #ifdef UNIV_SYNC_DEBUG
2193@@ -718,7 +817,7 @@
eccb488f
ER		 2194 ut_ad(lock);
2195 ut_ad(rw_lock_validate(lock));
2196
4b771f0e
ER		 2197- mutex_enter(&(lock->mutex));
2198+ rw_lock_debug_mutex_enter();
eccb488f
ER		 2199
2200 info = UT_LIST_GET_FIRST(lock->debug_list);
2201
4b771f0e 2202@@ -728,7 +827,7 @@
eccb488f
ER		 2203 && (info->pass == 0)
2204 && (info->lock_type == lock_type)) {
2205
4b771f0e
ER		 2206- mutex_exit(&(lock->mutex));
2207+ rw_lock_debug_mutex_exit();
eccb488f
ER		 2208 /* Found! */
2209
2210 return(TRUE);
4b771f0e 2211@@ -736,7 +835,7 @@
eccb488f
ER		 2212
2213 info = UT_LIST_GET_NEXT(list, info);
2214 }
4b771f0e
ER		 2215- mutex_exit(&(lock->mutex));
2216+ rw_lock_debug_mutex_exit();
eccb488f
ER		 2217
2218 return(FALSE);
2219 }
4b771f0e 2220@@ -758,22 +857,18 @@
eccb488f
ER		 2221 ut_ad(lock);
2222 ut_ad(rw_lock_validate(lock));
2223
4b771f0e
ER		 2224- mutex_enter(&(lock->mutex));
2225-
eccb488f 2226 if (lock_type == RW_LOCK_SHARED) {
4b771f0e
ER		 2227- if (lock->reader_count > 0) {
2228+ if (rw_lock_get_reader_count(lock) > 0) {
eccb488f
ER		 2229 ret = TRUE;
2230 }
2231 } else if (lock_type == RW_LOCK_EX) {
2232- if (lock->writer == RW_LOCK_EX) {
2233+ if (rw_lock_get_writer(lock) == RW_LOCK_EX) {
2234 ret = TRUE;
2235 }
2236 } else {
2237 ut_error;
2238 }
2239
4b771f0e
ER		 2240- mutex_exit(&(lock->mutex));
2241-
eccb488f
ER		 2242 return(ret);
2243 }
4b771f0e
ER		 2244
2245@@ -801,11 +896,10 @@
eccb488f
ER		 2246
2247 count++;
2248
2249+#ifndef HAVE_ATOMIC_BUILTINS
2250 mutex_enter(&(lock->mutex));
4b771f0e
ER		 2251-
2252- if ((rw_lock_get_writer(lock) != RW_LOCK_NOT_LOCKED)
2253- || (rw_lock_get_reader_count(lock) != 0)
eccb488f 2254- || (rw_lock_get_waiters(lock) != 0)) {
4b771f0e
ER		 2255+#endif
2256+ if (lock->lock_word != X_LOCK_DECR) {
eccb488f
ER		 2257
2258 fprintf(stderr, "RW-LOCK: %p ", lock);
2259
4b771f0e
ER		 2260@@ -821,8 +915,10 @@
2261 info = UT_LIST_GET_NEXT(list, info);
eccb488f
ER		 2262 }
2263 }
4b771f0e 2264-
eccb488f
ER		 2265+#ifndef HAVE_ATOMIC_BUILTINS
2266 mutex_exit(&(lock->mutex));
2267+#endif
4b771f0e 2268+
eccb488f
ER		 2269 lock = UT_LIST_GET_NEXT(list, lock);
2270 }
2271
4b771f0e
ER		 2272@@ -845,9 +941,10 @@
2273 "RW-LATCH INFO\n"
2274 "RW-LATCH: %p ", lock);
eccb488f 2275
4b771f0e
ER		 2276- if ((rw_lock_get_writer(lock) != RW_LOCK_NOT_LOCKED)
2277- || (rw_lock_get_reader_count(lock) != 0)
eccb488f 2278- || (rw_lock_get_waiters(lock) != 0)) {
4b771f0e
ER		 2279+#ifndef HAVE_ATOMIC_BUILTINS
2280+ mutex_enter(&(lock->mutex));
2281+#endif
2282+ if (lock->lock_word != X_LOCK_DECR) {
2283
2284 if (rw_lock_get_waiters(lock)) {
2285 fputs(" Waiters for the lock exist\n", stderr);
2286@@ -861,6 +958,9 @@
2287 info = UT_LIST_GET_NEXT(list, info);
eccb488f 2288 }
4b771f0e
ER		 2289 }
2290+#ifndef HAVE_ATOMIC_BUILTINS
2291+ mutex_exit(&(lock->mutex));
2292+#endif
2293 }
2294
2295 /*************************************************************************
2296@@ -909,14 +1009,11 @@
eccb488f
ER		 2297 lock = UT_LIST_GET_FIRST(rw_lock_list);
2298
2299 while (lock != NULL) {
4b771f0e 2300- mutex_enter(rw_lock_get_mutex(lock));
eccb488f 2301
4b771f0e
ER		 2302- if ((rw_lock_get_writer(lock) != RW_LOCK_NOT_LOCKED)
2303- || (rw_lock_get_reader_count(lock) != 0)) {
2304+ if (lock->lock_word != X_LOCK_DECR) {
eccb488f
ER		 2305 count++;
2306 }
2307
4b771f0e 2308- mutex_exit(rw_lock_get_mutex(lock));
eccb488f
ER		 2309 lock = UT_LIST_GET_NEXT(list, lock);
2310 }
2311
4b771f0e
ER		 2312diff -ruN a/innobase/sync/sync0sync.c b/innobase/sync/sync0sync.c
2313--- a/innobase/sync/sync0sync.c 2009-10-22 15:15:05.000000000 +0900
2314+++ b/innobase/sync/sync0sync.c 2009-10-22 15:18:44.000000000 +0900
2315@@ -1,8 +1,31 @@
2316+/*****************************************************************************
2317+
2318+Copyright (c) 1995, 2009, Innobase Oy. All Rights Reserved.
2319+Copyright (c) 2008, Google Inc.
2320+
2321+Portions of this file contain modifications contributed and copyrighted by
2322+Google, Inc. Those modifications are gratefully acknowledged and are described
2323+briefly in the InnoDB documentation. The contributions by Google are
2324+incorporated with their permission, and subject to the conditions contained in
2325+the file COPYING.Google.
2326+
2327+This program is free software; you can redistribute it and/or modify it under
2328+the terms of the GNU General Public License as published by the Free Software
2329+Foundation; version 2 of the License.
2330+
2331+This program is distributed in the hope that it will be useful, but WITHOUT
2332+ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
2333+FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
2334+
2335+You should have received a copy of the GNU General Public License along with
2336+this program; if not, write to the Free Software Foundation, Inc., 59 Temple
2337+Place, Suite 330, Boston, MA 02111-1307 USA
2338+
2339+*****************************************************************************/
2340+
2341 /******************************************************
2342 Mutex, the basic synchronization primitive
2343
2344-(c) 1995 Innobase Oy
2345-
2346 Created 9/5/1995 Heikki Tuuri
2347 *******************************************************/
2348
2349@@ -140,17 +163,12 @@
2350
2351 ulint sync_dummy = 0;
2352
2353-/* The number of system calls made in this module. Intended for performance
2354-monitoring. */
2355-
2356-ulint mutex_system_call_count = 0;
2357-
2358 /* Number of spin waits on mutexes: for performance monitoring */
2359
2360-ulint mutex_spin_round_count = 0;
2361-ulint mutex_spin_wait_count = 0;
2362-ulint mutex_os_wait_count = 0;
2363-ulint mutex_exit_count = 0;
2364+ib_longlong mutex_spin_round_count = 0;
2365+ib_longlong mutex_spin_wait_count = 0;
2366+ib_longlong mutex_os_wait_count = 0;
2367+ib_longlong mutex_exit_count = 0;
2368
2369 /* The global array of wait cells for implementation of the database's own
2370 mutexes and read-write locks */
2371@@ -240,6 +258,8 @@
2372 {
2373 #if defined(_WIN32) && defined(UNIV_CAN_USE_X86_ASSEMBLER)
2374 mutex_reset_lock_word(mutex);
2375+#elif defined(HAVE_ATOMIC_BUILTINS)
2376+ mutex_reset_lock_word(mutex);
2377 #else
2378 os_fast_mutex_init(&(mutex->os_fast_mutex));
2379 mutex->lock_word = 0;
2380@@ -325,7 +345,9 @@
2381
2382 os_event_free(mutex->event);
2383
2384-#if !defined(_WIN32) || !defined(UNIV_CAN_USE_X86_ASSEMBLER)
2385+#if defined(_WIN32) && defined(UNIV_CAN_USE_X86_ASSEMBLER)
2386+#elif defined(HAVE_ATOMIC_BUILTINS)
2387+#else
2388 os_fast_mutex_free(&(mutex->os_fast_mutex));
2389 #endif
2390 /* If we free the mutex protecting the mutex list (freeing is
2391@@ -421,6 +443,12 @@
2392 #endif /* UNIV_DEBUG && !UNIV_HOTBACKUP */
2393 ut_ad(mutex);
2394
2395+ /* This update is not thread safe, but we don't mind if the count
2396+ isn't exact. Moved out of ifdef that follows because we are willing
2397+ to sacrifice the cost of counting this as the data is valuable.
2398+ Count the number of calls to mutex_spin_wait. */
2399+ mutex_spin_wait_count++;
2400+
2401 mutex_loop:
2402
2403 i = 0;
2404@@ -433,7 +461,6 @@
2405
2406 spin_loop:
2407 #if defined UNIV_DEBUG && !defined UNIV_HOTBACKUP
2408- mutex_spin_wait_count++;
2409 mutex->count_spin_loop++;
2410 #endif /* UNIV_DEBUG && !UNIV_HOTBACKUP */
2411
2412@@ -502,8 +529,6 @@
2413 sync_array_reserve_cell(sync_primary_wait_array, mutex,
2414 SYNC_MUTEX, file_name, line, &index);
2415
2416- mutex_system_call_count++;
2417-
2418 /* The memory order of the array reservation and the change in the
2419 waiters field is important: when we suspend a thread, we first
2420 reserve the cell and then set waiters field to 1. When threads are
2421@@ -551,7 +576,6 @@
2422 mutex->cfile_name, (ulong) mutex->cline, (ulong) i);
2423 #endif
2424
2425- mutex_system_call_count++;
2426 mutex_os_wait_count++;
2427
2428 #ifndef UNIV_HOTBACKUP
2429@@ -1368,20 +1392,31 @@
2430 FILE* file) /* in: file where to print */
2431 {
2432 #ifdef UNIV_SYNC_DEBUG
2433- fprintf(stderr, "Mutex exits %lu, rws exits %lu, rwx exits %lu\n",
2434+ fprintf(file, "Mutex exits %llu, rws exits %llu, rwx exits %llu\n",
2435 mutex_exit_count, rw_s_exit_count, rw_x_exit_count);
2436 #endif
2437
2438 fprintf(file,
2439-"Mutex spin waits %lu, rounds %lu, OS waits %lu\n"
2440-"RW-shared spins %lu, OS waits %lu; RW-excl spins %lu, OS waits %lu\n",
2441- (ulong) mutex_spin_wait_count,
2442- (ulong) mutex_spin_round_count,
2443- (ulong) mutex_os_wait_count,
2444- (ulong) rw_s_spin_wait_count,
2445- (ulong) rw_s_os_wait_count,
2446- (ulong) rw_x_spin_wait_count,
2447- (ulong) rw_x_os_wait_count);
2448+ "Mutex spin waits %llu, rounds %llu, OS waits %llu\n"
2449+ "RW-shared spins %llu, OS waits %llu;"
2450+ " RW-excl spins %llu, OS waits %llu\n",
2451+ mutex_spin_wait_count,
2452+ mutex_spin_round_count,
2453+ mutex_os_wait_count,
2454+ rw_s_spin_wait_count,
2455+ rw_s_os_wait_count,
2456+ rw_x_spin_wait_count,
2457+ rw_x_os_wait_count);
2458+
2459+ fprintf(file,
2460+ "Spin rounds per wait: %.2f mutex, %.2f RW-shared, "
2461+ "%.2f RW-excl\n",
2462+ (double) mutex_spin_round_count /
2463+ (mutex_spin_wait_count ? mutex_spin_wait_count : 1),
2464+ (double) rw_s_spin_round_count /
2465+ (rw_s_spin_wait_count ? rw_s_spin_wait_count : 1),
2466+ (double) rw_x_spin_round_count /
2467+ (rw_x_spin_wait_count ? rw_x_spin_wait_count : 1));
2468 }
2469
2470 /***********************************************************************
36400f81 2471diff -ruN /dev/null b/patch_info/innodb_rw_lock.info
45532174 2472--- /dev/null 1970-01-01 09:00:00.000000000 +0900
4b771f0e 2473+++ b/patch_info/innodb_rw_lock.info 2009-10-22 15:18:30.000000000 +0900
eccb488f
ER		 2474@@ -0,0 +1,6 @@
2475+File=innodb_rw_lock.patch
4b771f0e 2476+Name=Fix of InnoDB rw_locks ported from InnoDB Plugin
eccb488f 2477+Version=1.0
4b771f0e
ER		 2478+Author=InnoBase Oy.
2479+License=GPL
eccb488f 2480+Comment=
MySQL: a very fast and reliable SQL database engine
This page took 0.353684 seconds and 4 git commands to generate.