mysql-innodb_rw_lock.patch

   1 diff -ruN a/innobase/btr/btr0cur.c b/innobase/btr/btr0cur.c
   2 --- a/innobase/btr/btr0cur.c    2009-10-22 15:15:05.000000000 +0900
   3 +++ b/innobase/btr/btr0cur.c    2009-10-22 15:18:44.000000000 +0900
   4 @@ -313,7 +313,7 @@
   5  #ifdef UNIV_SEARCH_PERF_STAT
   6         info->n_searches++;
   7  #endif
   8 -       if (btr_search_latch.writer == RW_LOCK_NOT_LOCKED
   9 +       if (rw_lock_get_writer(&btr_search_latch) == RW_LOCK_NOT_LOCKED
  10                 && latch_mode <= BTR_MODIFY_LEAF && info->last_hash_succ
  11                 && !estimate
  12  #ifdef PAGE_CUR_LE_OR_EXTENDS
  13 diff -ruN a/innobase/btr/btr0sea.c b/innobase/btr/btr0sea.c
  14 --- a/innobase/btr/btr0sea.c    2009-10-22 15:15:05.000000000 +0900
  15 +++ b/innobase/btr/btr0sea.c    2009-10-22 15:18:44.000000000 +0900
  16 @@ -773,8 +773,8 @@
  17                 rw_lock_s_lock(&btr_search_latch);
  18         }
  19
  20 -       ut_ad(btr_search_latch.writer != RW_LOCK_EX);
  21 -       ut_ad(btr_search_latch.reader_count > 0);
  22 +       ut_ad(rw_lock_get_writer(&btr_search_latch) != RW_LOCK_EX);
  23 +       ut_ad(rw_lock_get_reader_count(&btr_search_latch) > 0);
  24
  25         rec = ha_search_and_get_data(btr_search_sys->hash_index, fold);
  26
  27 diff -ruN a/innobase/buf/buf0buf.c b/innobase/buf/buf0buf.c
  28 --- a/innobase/buf/buf0buf.c    2009-10-22 15:15:05.000000000 +0900
  29 +++ b/innobase/buf/buf0buf.c    2009-10-22 15:18:44.000000000 +0900
  30 @@ -1292,7 +1292,7 @@
  31
  32         if (mode == BUF_GET_NOWAIT) {
  33                 if (rw_latch == RW_S_LATCH) {
  34 -                       success = rw_lock_s_lock_func_nowait(&(block->lock),
  35 +                       success = rw_lock_s_lock_nowait(&(block->lock),
  36                                                                 file, line);
  37                         fix_type = MTR_MEMO_PAGE_S_FIX;
  38                 } else {
  39 @@ -1442,7 +1442,7 @@
  40         ut_ad(!ibuf_inside() || ibuf_page(block->space, block->offset));
  41
  42         if (rw_latch == RW_S_LATCH) {
  43 -               success = rw_lock_s_lock_func_nowait(&(block->lock),
  44 +               success = rw_lock_s_lock_nowait(&(block->lock),
  45                                                                 file, line);
  46                 fix_type = MTR_MEMO_PAGE_S_FIX;
  47         } else {
  48 @@ -1596,7 +1596,7 @@
  49         ut_ad(!ibuf_inside() || (mode == BUF_KEEP_OLD));
  50
  51         if (rw_latch == RW_S_LATCH) {
  52 -               success = rw_lock_s_lock_func_nowait(&(block->lock),
  53 +               success = rw_lock_s_lock_nowait(&(block->lock),
  54                                                                 file, line);
  55                 fix_type = MTR_MEMO_PAGE_S_FIX;
  56         } else {
  57 diff -ruN a/innobase/include/buf0buf.ic b/innobase/include/buf0buf.ic
  58 --- a/innobase/include/buf0buf.ic       2009-10-22 15:15:05.000000000 +0900
  59 +++ b/innobase/include/buf0buf.ic       2009-10-22 16:12:25.000000000 +0900
  60 @@ -523,7 +523,7 @@
  61  #ifdef UNIV_SYNC_DEBUG
  62         ibool   ret;
  63
  64 -       ret = rw_lock_s_lock_func_nowait(&(block->debug_latch), file, line);
  65 +       ret = rw_lock_s_lock_nowait(&(block->debug_latch), file, line);
  66
  67         ut_ad(ret == TRUE);
  68         ut_ad(mutex_own(&block->mutex));
  69 diff -ruN a/innobase/include/os0sync.h b/innobase/include/os0sync.h
  70 --- a/innobase/include/os0sync.h        2009-09-10 04:02:59.000000000 +0900
  71 +++ b/innobase/include/os0sync.h        2009-10-22 15:18:44.000000000 +0900
  72 @@ -1,11 +1,35 @@
  73 +/*****************************************************************************
  74 +
  75 +Copyright (c) 1995, 2009, Innobase Oy. All Rights Reserved.
  76 +Copyright (c) 2008, Google Inc.
  77 +
  78 +Portions of this file contain modifications contributed and copyrighted by
  79 +Google, Inc. Those modifications are gratefully acknowledged and are described
  80 +briefly in the InnoDB documentation. The contributions by Google are
  81 +incorporated with their permission, and subject to the conditions contained in
  82 +the file COPYING.Google.
  83 +
  84 +This program is free software; you can redistribute it and/or modify it under
  85 +the terms of the GNU General Public License as published by the Free Software
  86 +Foundation; version 2 of the License.
  87 +
  88 +This program is distributed in the hope that it will be useful, but WITHOUT
  89 +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  90 +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
  91 +
  92 +You should have received a copy of the GNU General Public License along with
  93 +this program; if not, write to the Free Software Foundation, Inc., 59 Temple
  94 +Place, Suite 330, Boston, MA 02111-1307 USA
  95 +
  96 +*****************************************************************************/
  97 +
  98  /******************************************************
  99  The interface to the operating system
 100  synchronization primitives.
 101
 102 -(c) 1995 Innobase Oy
 103 -
 104  Created 9/6/1995 Heikki Tuuri
 105  *******************************************************/
 106 +
 107  #ifndef os0sync_h
 108  #define os0sync_h
 109
 110 @@ -261,6 +285,23 @@
 111  /*===============*/
 112         os_fast_mutex_t*        fast_mutex);    /* in: mutex to free */
 113
 114 +#ifdef HAVE_ATOMIC_BUILTINS
 115 +/**************************************************************
 116 +Atomic compare-and-swap for InnoDB. Currently requires GCC atomic builtins.
 117 +Returns true if swapped, ptr is pointer to target, old_val is value to
 118 +compare to, new_val is the value to swap in. */
 119 +#define os_compare_and_swap(ptr, old_val, new_val) \
 120 +       __sync_bool_compare_and_swap(ptr, old_val, new_val)
 121 +
 122 +/**************************************************************
 123 +Atomic increment for InnoDB. Currently requires GCC atomic builtins.
 124 +Returns the resulting value, ptr is pointer to target, amount is the
 125 +amount of increment. */
 126 +#define os_atomic_increment(ptr, amount) \
 127 +       __sync_add_and_fetch(ptr, amount)
 128 +
 129 +#endif /* HAVE_ATOMIC_BUILTINS */
 130 +
 131  #ifndef UNIV_NONINL
 132  #include "os0sync.ic"
 133  #endif
 134 diff -ruN a/innobase/include/sync0rw.h b/innobase/include/sync0rw.h
 135 --- a/innobase/include/sync0rw.h        2009-09-10 04:02:59.000000000 +0900
 136 +++ b/innobase/include/sync0rw.h        2009-10-22 15:18:44.000000000 +0900
 137 @@ -1,8 +1,31 @@
 138 +/*****************************************************************************
 139 +
 140 +Copyright (c) 1995, 2009, Innobase Oy. All Rights Reserved.
 141 +Copyright (c) 2008, Google Inc.
 142 +
 143 +Portions of this file contain modifications contributed and copyrighted by
 144 +Google, Inc. Those modifications are gratefully acknowledged and are described
 145 +briefly in the InnoDB documentation. The contributions by Google are
 146 +incorporated with their permission, and subject to the conditions contained in
 147 +the file COPYING.Google.
 148 +
 149 +This program is free software; you can redistribute it and/or modify it under
 150 +the terms of the GNU General Public License as published by the Free Software
 151 +Foundation; version 2 of the License.
 152 +
 153 +This program is distributed in the hope that it will be useful, but WITHOUT
 154 +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 155 +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
 156 +
 157 +You should have received a copy of the GNU General Public License along with
 158 +this program; if not, write to the Free Software Foundation, Inc., 59 Temple
 159 +Place, Suite 330, Boston, MA 02111-1307 USA
 160 +
 161 +*****************************************************************************/
 162 +
 163  /******************************************************
 164  The read-write lock (for threads, not for database transactions)
 165
 166 -(c) 1995 Innobase Oy
 167 -
 168  Created 9/11/1995 Heikki Tuuri
 169  *******************************************************/
 170
 171 @@ -24,6 +47,12 @@
 172  #define        RW_X_LATCH      2
 173  #define        RW_NO_LATCH     3
 174
 175 +/* We decrement lock_word by this amount for each x_lock. It is also the
 176 +start value for the lock_word, meaning that it limits the maximum number
 177 +of concurrent read locks before the rw_lock breaks. The current value of
 178 +0x00100000 allows 1,048,575 concurrent readers and 2047 recursive writers.*/
 179 +#define X_LOCK_DECR            0x00100000
 180 +
 181  typedef struct rw_lock_struct          rw_lock_t;
 182  #ifdef UNIV_SYNC_DEBUG
 183  typedef struct rw_lock_debug_struct    rw_lock_debug_t;
 184 @@ -47,14 +76,14 @@
 185                                         there may be waiters for the event */
 186  #endif /* UNIV_SYNC_DEBUG */
 187
 188 -extern ulint   rw_s_system_call_count;
 189 -extern ulint   rw_s_spin_wait_count;
 190 -extern ulint   rw_s_exit_count;
 191 -extern ulint   rw_s_os_wait_count;
 192 -extern ulint   rw_x_system_call_count;
 193 -extern ulint   rw_x_spin_wait_count;
 194 -extern ulint   rw_x_os_wait_count;
 195 -extern ulint   rw_x_exit_count;
 196 +extern ib_longlong     rw_s_spin_wait_count;
 197 +extern ib_longlong     rw_s_spin_round_count;
 198 +extern ib_longlong     rw_s_exit_count;
 199 +extern ib_longlong     rw_s_os_wait_count;
 200 +extern ib_longlong     rw_x_spin_wait_count;
 201 +extern ib_longlong     rw_x_spin_round_count;
 202 +extern ib_longlong     rw_x_os_wait_count;
 203 +extern ib_longlong     rw_x_exit_count;
 204
 205  /**********************************************************************
 206  Creates, or rather, initializes an rw-lock object in a specified memory
 207 @@ -116,8 +145,22 @@
 208  NOTE! The following macros should be used in rw s-locking, not the
 209  corresponding function. */
 210
 211 -#define rw_lock_s_lock_nowait(M)    rw_lock_s_lock_func_nowait(\
 212 -                                            (M), __FILE__, __LINE__)
 213 +#define rw_lock_s_lock_nowait(M, F, L)    rw_lock_s_lock_low(\
 214 +                                         (M), 0, (F), (L))
 215 +/**********************************************************************
 216 +Low-level function which tries to lock an rw-lock in s-mode. Performs no
 217 +spinning. */
 218 +UNIV_INLINE
 219 +ibool
 220 +rw_lock_s_lock_low(
 221 +/*===============*/
 222 +                               /* out: TRUE if success */
 223 +       rw_lock_t*      lock,   /* in: pointer to rw-lock */
 224 +       ulint           pass __attribute__((unused)),
 225 +                               /* in: pass value; != 0, if the lock will be
 226 +                               passed to another thread to unlock */
 227 +       const char*     file_name, /* in: file name where lock requested */
 228 +       ulint           line);  /* in: line where requested */
 229  /**********************************************************************
 230  NOTE! Use the corresponding macro, not directly this function, except if
 231  you supply the file name and line number. Lock an rw-lock in shared mode
 232 @@ -135,18 +178,6 @@
 233         const char*     file_name,/* in: file name where lock requested */
 234         ulint           line);  /* in: line where requested */
 235  /**********************************************************************
 236 -NOTE! Use the corresponding macro, not directly this function, except if
 237 -you supply the file name and line number. Lock an rw-lock in shared mode
 238 -for the current thread if the lock can be acquired immediately. */
 239 -UNIV_INLINE
 240 -ibool
 241 -rw_lock_s_lock_func_nowait(
 242 -/*=======================*/
 243 -                               /* out: TRUE if success */
 244 -        rw_lock_t*     lock,   /* in: pointer to rw-lock */
 245 -       const char*     file_name,/* in: file name where lock requested */
 246 -       ulint           line);  /* in: line where requested */
 247 -/**********************************************************************
 248  NOTE! Use the corresponding macro, not directly this function! Lock an
 249  rw-lock in exclusive mode for the current thread if the lock can be
 250  obtained immediately. */
 251 @@ -338,6 +369,41 @@
 252  rw_lock_get_reader_count(
 253  /*=====================*/
 254         rw_lock_t*      lock);
 255 +/**********************************************************************
 256 +Decrements lock_word the specified amount if it is greater than 0.
 257 +This is used by both s_lock and x_lock operations. */
 258 +UNIV_INLINE
 259 +ibool
 260 +rw_lock_lock_word_decr(
 261 +/*===================*/
 262 +                                       /* out: TRUE if decr occurs */
 263 +       rw_lock_t*      lock,           /* in: rw-lock */
 264 +       ulint           amount);        /* in: amount to decrement */
 265 +/**********************************************************************
 266 +Increments lock_word the specified amount and returns new value. */
 267 +UNIV_INLINE
 268 +lint
 269 +rw_lock_lock_word_incr(
 270 +/*===================*/
 271 +                                       /* out: TRUE if decr occurs */
 272 +       rw_lock_t*      lock,
 273 +       ulint           amount);        /* in: rw-lock */
 274 +/**********************************************************************
 275 +This function sets the lock->writer_thread and lock->recursive fields.
 276 +For platforms where we are using atomic builtins instead of lock->mutex
 277 +it sets the lock->writer_thread field using atomics to ensure memory
 278 +ordering. Note that it is assumed that the caller of this function
 279 +effectively owns the lock i.e.: nobody else is allowed to modify
 280 +lock->writer_thread at this point in time.
 281 +The protocol is that lock->writer_thread MUST be updated BEFORE the
 282 +lock->recursive flag is set. */
 283 +UNIV_INLINE
 284 +void
 285 +rw_lock_set_writer_id_and_recursion_flag(
 286 +/*=====================================*/
 287 +       rw_lock_t*      lock,           /* in/out: lock to work on */
 288 +       ibool           recursive);     /* in: TRUE if recursion
 289 +                                       allowed */
 290  #ifdef UNIV_SYNC_DEBUG
 291  /**********************************************************************
 292  Checks if the thread has locked the rw-lock in the specified mode, with
 293 @@ -417,47 +483,33 @@
 294  field. Then no new readers are allowed in. */
 295
 296  struct rw_lock_struct {
 297 +       volatile lint   lock_word;
 298 +                               /* Holds the state of the lock. */
 299 +       volatile ulint  waiters;/* 1: there are waiters */
 300 +       volatile ibool  recursive;/* Default value FALSE which means the lock
 301 +                               is non-recursive. The value is typically set
 302 +                               to TRUE making normal rw_locks recursive. In
 303 +                               case of asynchronous IO, when a non-zero
 304 +                               value of 'pass' is passed then we keep the
 305 +                               lock non-recursive.
 306 +                               This flag also tells us about the state of
 307 +                               writer_thread field. If this flag is set
 308 +                               then writer_thread MUST contain the thread
 309 +                               id of the current x-holder or wait-x thread.
 310 +                               This flag must be reset in x_unlock
 311 +                               functions before incrementing the lock_word */
 312 +       volatile os_thread_id_t writer_thread;
 313 +                               /* Thread id of writer thread. Is only
 314 +                               guaranteed to have sane and non-stale
 315 +                               value iff recursive flag is set. */
 316         os_event_t      event;  /* Used by sync0arr.c for thread queueing */
 317 -
 318 -#ifdef __WIN__
 319 -       os_event_t      wait_ex_event;  /* This windows specific event is
 320 -                               used by the thread which has set the
 321 -                               lock state to RW_LOCK_WAIT_EX. The
 322 -                               rw_lock design guarantees that this
 323 -                               thread will be the next one to proceed
 324 -                               once the current the event gets
 325 -                               signalled. See LEMMA 2 in sync0sync.c */
 326 -#endif
 327 -
 328 -       ulint   reader_count;   /* Number of readers who have locked this
 329 -                               lock in the shared mode */
 330 -       ulint   writer;         /* This field is set to RW_LOCK_EX if there
 331 -                               is a writer owning the lock (in exclusive
 332 -                               mode), RW_LOCK_WAIT_EX if a writer is
 333 -                               queueing for the lock, and
 334 -                               RW_LOCK_NOT_LOCKED, otherwise. */
 335 -       os_thread_id_t  writer_thread;
 336 -                               /* Thread id of a possible writer thread */
 337 -       ulint   writer_count;   /* Number of times the same thread has
 338 -                               recursively locked the lock in the exclusive
 339 -                               mode */
 340 +       os_event_t      wait_ex_event;
 341 +                               /* Event for next-writer to wait on. A thread
 342 +                               must decrement lock_word before waiting. */
 343 +#ifndef HAVE_ATOMIC_BUILTINS
 344         mutex_t mutex;          /* The mutex protecting rw_lock_struct */
 345 -       ulint   pass;           /* Default value 0. This is set to some
 346 -                               value != 0 given by the caller of an x-lock
 347 -                               operation, if the x-lock is to be passed to
 348 -                               another thread to unlock (which happens in
 349 -                               asynchronous i/o). */
 350 -       ulint   waiters;        /* This ulint is set to 1 if there are
 351 -                               waiters (readers or writers) in the global
 352 -                               wait array, waiting for this rw_lock.
 353 -                               Otherwise, == 0. */
 354 -       ibool   writer_is_wait_ex;
 355 -                               /* This is TRUE if the writer field is
 356 -                               RW_LOCK_WAIT_EX; this field is located far
 357 -                               from the memory update hotspot fields which
 358 -                               are at the start of this struct, thus we can
 359 -                               peek this field without causing much memory
 360 -                               bus traffic */
 361 +#endif /* HAVE_ATOMIC_BUILTINS */
 362 +
 363         UT_LIST_NODE_T(rw_lock_t) list;
 364                                 /* All allocated rw locks are put into a
 365                                 list */
 366 @@ -465,15 +517,23 @@
 367         UT_LIST_BASE_NODE_T(rw_lock_debug_t) debug_list;
 368                                 /* In the debug version: pointer to the debug
 369                                 info list of the lock */
 370 +       ulint   level;          /* Level in the global latching order. */
 371  #endif /* UNIV_SYNC_DEBUG */
 372 -       ulint   level;          /* Level in the global latching
 373 -                               order; default SYNC_LEVEL_NONE */
 374 +       ulint count_os_wait;    /* Count of os_waits. May not be accurate */
 375         const char*     cfile_name;/* File name where lock created */
 376 -       ulint   cline;          /* Line where created */
 377 +        /* last s-lock file/line is not guaranteed to be correct */
 378         const char*     last_s_file_name;/* File name where last s-locked */
 379         const char*     last_x_file_name;/* File name where last x-locked */
 380 -       ulint   last_s_line;    /* Line number where last time s-locked */
 381 -       ulint   last_x_line;    /* Line number where last time x-locked */
 382 +       ibool           writer_is_wait_ex;
 383 +                               /* This is TRUE if the writer field is
 384 +                               RW_LOCK_WAIT_EX; this field is located far
 385 +                               from the memory update hotspot fields which
 386 +                               are at the start of this struct, thus we can
 387 +                               peek this field without causing much memory
 388 +                               bus traffic */
 389 +       unsigned        cline:14;       /* Line where created */
 390 +       unsigned        last_s_line:14; /* Line number where last time s-locked */
 391 +       unsigned        last_x_line:14; /* Line number where last time x-locked */
 392         ulint   magic_n;
 393  };
 394
 395 diff -ruN a/innobase/include/sync0rw.ic b/innobase/include/sync0rw.ic
 396 --- a/innobase/include/sync0rw.ic       2009-09-10 04:02:59.000000000 +0900
 397 +++ b/innobase/include/sync0rw.ic       2009-10-22 15:18:44.000000000 +0900
 398 @@ -1,8 +1,31 @@
 399 +/*****************************************************************************
 400 +
 401 +Copyright (c) 1995, 2009, Innobase Oy. All Rights Reserved.
 402 +Copyright (c) 2008, Google Inc.
 403 +
 404 +Portions of this file contain modifications contributed and copyrighted by
 405 +Google, Inc. Those modifications are gratefully acknowledged and are described
 406 +briefly in the InnoDB documentation. The contributions by Google are
 407 +incorporated with their permission, and subject to the conditions contained in
 408 +the file COPYING.Google.
 409 +
 410 +This program is free software; you can redistribute it and/or modify it under
 411 +the terms of the GNU General Public License as published by the Free Software
 412 +Foundation; version 2 of the License.
 413 +
 414 +This program is distributed in the hope that it will be useful, but WITHOUT
 415 +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 416 +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
 417 +
 418 +You should have received a copy of the GNU General Public License along with
 419 +this program; if not, write to the Free Software Foundation, Inc., 59 Temple
 420 +Place, Suite 330, Boston, MA 02111-1307 USA
 421 +
 422 +*****************************************************************************/
 423 +
 424  /******************************************************
 425  The read-write lock (for threads)
 426
 427 -(c) 1995 Innobase Oy
 428 -
 429  Created 9/11/1995 Heikki Tuuri
 430  *******************************************************/
 431
 432 @@ -49,53 +72,88 @@
 433  ulint
 434  rw_lock_get_waiters(
 435  /*================*/
 436 -       rw_lock_t*      lock)
 437 +                               /* out: 1 if waiters, 0 otherwise */
 438 +       rw_lock_t*      lock)   /* in: rw-lock */
 439  {
 440         return(lock->waiters);
 441  }
 442 +
 443 +/************************************************************************
 444 +Sets lock->waiters to 1. It is not an error if lock->waiters is already
 445 +1. On platforms where ATOMIC builtins are used this function enforces a
 446 +memory barrier. */
 447  UNIV_INLINE
 448  void
 449 -rw_lock_set_waiters(
 450 -/*================*/
 451 -       rw_lock_t*      lock,
 452 -       ulint           flag)
 453 +rw_lock_set_waiter_flag(
 454 +/*====================*/
 455 +       rw_lock_t*      lock)   /* in: rw-lock */
 456  {
 457 -       lock->waiters = flag;
 458 +#ifdef HAVE_ATOMIC_BUILTINS
 459 +       os_compare_and_swap(&lock->waiters, 0, 1);
 460 +#else /* HAVE_ATOMIC_BUILTINS */
 461 +       lock->waiters = 1;
 462 +#endif /* HAVE_ATOMIC_BUILTINS */
 463  }
 464 +
 465 +/************************************************************************
 466 +Resets lock->waiters to 0. It is not an error if lock->waiters is already
 467 +0. On platforms where ATOMIC builtins are used this function enforces a
 468 +memory barrier. */
 469  UNIV_INLINE
 470 -ulint
 471 -rw_lock_get_writer(
 472 -/*===============*/
 473 -       rw_lock_t*      lock)
 474 +void
 475 +rw_lock_reset_waiter_flag(
 476 +/*======================*/
 477 +       rw_lock_t*      lock)   /* in: rw-lock */
 478  {
 479 -       return(lock->writer);
 480 +#ifdef HAVE_ATOMIC_BUILTINS
 481 +       os_compare_and_swap(&lock->waiters, 1, 0);
 482 +#else /* HAVE_ATOMIC_BUILTINS */
 483 +       lock->waiters = 0;
 484 +#endif /* HAVE_ATOMIC_BUILTINS */
 485  }
 486 +
 487 +/**********************************************************************
 488 +Returns the write-status of the lock - this function made more sense
 489 +with the old rw_lock implementation. */
 490  UNIV_INLINE
 491 -void
 492 -rw_lock_set_writer(
 493 +ulint
 494 +rw_lock_get_writer(
 495  /*===============*/
 496 -       rw_lock_t*      lock,
 497 -       ulint           flag)
 498 +       rw_lock_t*      lock)
 499  {
 500 -       lock->writer = flag;
 501 +       lint lock_word = lock->lock_word;
 502 +       if(lock_word > 0) {
 503 +               /* return NOT_LOCKED in s-lock state, like the writer
 504 +               member of the old lock implementation. */
 505 +               return(RW_LOCK_NOT_LOCKED);
 506 +       } else if (((-lock_word) % X_LOCK_DECR) == 0) {
 507 +               return(RW_LOCK_EX);
 508 +       } else {
 509 +                ut_ad(lock_word > -X_LOCK_DECR);
 510 +               return(RW_LOCK_WAIT_EX);
 511 +       }
 512  }
 513 +
 514 +/**********************************************************************
 515 +Returns number of readers. */
 516  UNIV_INLINE
 517  ulint
 518  rw_lock_get_reader_count(
 519  /*=====================*/
 520         rw_lock_t*      lock)
 521  {
 522 -       return(lock->reader_count);
 523 -}
 524 -UNIV_INLINE
 525 -void
 526 -rw_lock_set_reader_count(
 527 -/*=====================*/
 528 -       rw_lock_t*      lock,
 529 -       ulint           count)
 530 -{
 531 -       lock->reader_count = count;
 532 +       lint lock_word = lock->lock_word;
 533 +       if(lock_word > 0) {
 534 +               /* s-locked, no x-waiters */
 535 +               return(X_LOCK_DECR - lock_word);
 536 +       } else if (lock_word < 0 && lock_word > -X_LOCK_DECR) {
 537 +               /* s-locked, with x-waiters */
 538 +               return((ulint)(-lock_word));
 539 +       }
 540 +       return(0);
 541  }
 542 +
 543 +#ifndef HAVE_ATOMIC_BUILTINS
 544  UNIV_INLINE
 545  mutex_t*
 546  rw_lock_get_mutex(
 547 @@ -104,6 +162,7 @@
 548  {
 549         return(&(lock->mutex));
 550  }
 551 +#endif
 552
 553  /**********************************************************************
 554  Returns the value of writer_count for the lock. Does not reserve the lock
 555 @@ -115,7 +174,126 @@
 556                                 /* out: value of writer_count */
 557         rw_lock_t*      lock)   /* in: rw-lock */
 558  {
 559 -       return(lock->writer_count);
 560 +       lint lock_copy = lock->lock_word;
 561 +       /* If there is a reader, lock_word is not divisible by X_LOCK_DECR */
 562 +       if(lock_copy > 0 || (-lock_copy) % X_LOCK_DECR != 0) {
 563 +               return(0);
 564 +       }
 565 +       return(((-lock_copy) / X_LOCK_DECR) + 1);
 566 +}
 567 +
 568 +/**********************************************************************
 569 +Two different implementations for decrementing the lock_word of a rw_lock:
 570 +one for systems supporting atomic operations, one for others. This does
 571 +does not support recusive x-locks: they should be handled by the caller and
 572 +need not be atomic since they are performed by the current lock holder.
 573 +Returns true if the decrement was made, false if not. */
 574 +UNIV_INLINE
 575 +ibool
 576 +rw_lock_lock_word_decr(
 577 +/*===================*/
 578 +                               /* out: TRUE if decr occurs */
 579 +       rw_lock_t*      lock,   /* in: rw-lock */
 580 +       ulint           amount) /* in: amount of decrement */
 581 +{
 582 +
 583 +#ifdef HAVE_ATOMIC_BUILTINS
 584 +
 585 +        lint local_lock_word = lock->lock_word;
 586 +       while (local_lock_word > 0) {
 587 +               if(os_compare_and_swap(&(lock->lock_word),
 588 +                                       local_lock_word,
 589 +                                       local_lock_word - amount)) {
 590 +                       return(TRUE);
 591 +               }
 592 +               local_lock_word = lock->lock_word;
 593 +       }
 594 +       return(FALSE);
 595 +
 596 +#else /* HAVE_ATOMIC_BUILTINS */
 597 +
 598 +       ibool success = FALSE;
 599 +       mutex_enter(&(lock->mutex));
 600 +       if(lock->lock_word > 0) {
 601 +               lock->lock_word -= amount;
 602 +               success = TRUE;
 603 +       }
 604 +       mutex_exit(&(lock->mutex));
 605 +       return(success);
 606 +
 607 +#endif /* HAVE_ATOMIC_BUILTINS */
 608 +}
 609 +
 610 +/**********************************************************************
 611 +Two different implementations for incrementing the lock_word of a rw_lock:
 612 +one for systems supporting atomic operations, one for others.
 613 +Returns the value of lock_word after increment. */
 614 +UNIV_INLINE
 615 +lint
 616 +rw_lock_lock_word_incr(
 617 +/*===================*/
 618 +                               /* out: lock->lock_word after increment */
 619 +       rw_lock_t*      lock,   /* in: rw-lock */
 620 +       ulint           amount) /* in: amount of increment */
 621 +{
 622 +
 623 +#ifdef HAVE_ATOMIC_BUILTINS
 624 +
 625 +       return(os_atomic_increment(&(lock->lock_word), amount));
 626 +
 627 +#else /* HAVE_ATOMIC_BUILTINS */
 628 +
 629 +       lint local_lock_word;
 630 +
 631 +       mutex_enter(&(lock->mutex));
 632 +
 633 +       lock->lock_word += amount;
 634 +       local_lock_word = lock->lock_word;
 635 +
 636 +       mutex_exit(&(lock->mutex));
 637 +
 638 +        return(local_lock_word);
 639 +
 640 +#endif /* HAVE_ATOMIC_BUILTINS */
 641 +}
 642 +
 643 +/**********************************************************************
 644 +This function sets the lock->writer_thread and lock->recursive fields.
 645 +For platforms where we are using atomic builtins instead of lock->mutex
 646 +it sets the lock->writer_thread field using atomics to ensure memory
 647 +ordering. Note that it is assumed that the caller of this function
 648 +effectively owns the lock i.e.: nobody else is allowed to modify
 649 +lock->writer_thread at this point in time.
 650 +The protocol is that lock->writer_thread MUST be updated BEFORE the
 651 +lock->recursive flag is set. */
 652 +UNIV_INLINE
 653 +void
 654 +rw_lock_set_writer_id_and_recursion_flag(
 655 +/*=====================================*/
 656 +       rw_lock_t*      lock,           /* in/out: lock to work on */
 657 +       ibool           recursive)      /* in: TRUE if recursion
 658 +                                       allowed */
 659 +{
 660 +       os_thread_id_t  curr_thread     = os_thread_get_curr_id();
 661 +
 662 +#ifdef HAVE_ATOMIC_BUILTINS
 663 +       os_thread_id_t  local_thread;
 664 +       ibool           success;
 665 +
 666 +       local_thread = lock->writer_thread;
 667 +       success = os_compare_and_swap(&lock->writer_thread,
 668 +                                     local_thread, curr_thread);
 669 +       ut_a(success);
 670 +       lock->recursive = recursive;
 671 +
 672 +#else /* HAVE_ATOMIC_BUILTINS */
 673 +
 674 +       mutex_enter(&lock->mutex);
 675 +       lock->writer_thread = curr_thread;
 676 +       lock->recursive = recursive;
 677 +       mutex_exit(&lock->mutex);
 678 +
 679 +#endif /* HAVE_ATOMIC_BUILTINS */
 680  }
 681
 682  /**********************************************************************
 683 @@ -133,26 +311,21 @@
 684         const char*     file_name, /* in: file name where lock requested */
 685         ulint           line)   /* in: line where requested */
 686  {
 687 -#ifdef UNIV_SYNC_DEBUG
 688 -       ut_ad(mutex_own(rw_lock_get_mutex(lock)));
 689 -#endif /* UNIV_SYNC_DEBUG */
 690 -       /* Check if the writer field is free */
 691 -
 692 -       if (UNIV_LIKELY(lock->writer == RW_LOCK_NOT_LOCKED)) {
 693 -               /* Set the shared lock by incrementing the reader count */
 694 -               lock->reader_count++;
 695 +       /* TODO: study performance of UNIV_LIKELY branch prediction hints. */
 696 +       if (!rw_lock_lock_word_decr(lock, 1)) {
 697 +               /* Locking did not succeed */
 698 +               return(FALSE);
 699 +       }
 700
 701  #ifdef UNIV_SYNC_DEBUG
 702 -               rw_lock_add_debug_info(lock, pass, RW_LOCK_SHARED, file_name,
 703 -                                                                       line);
 704 +       rw_lock_add_debug_info(lock, pass, RW_LOCK_SHARED, file_name, line);
 705  #endif
 706 -               lock->last_s_file_name = file_name;
 707 -               lock->last_s_line = line;
 708 -
 709 -               return(TRUE);   /* locking succeeded */
 710 -       }
 711 +       /* These debugging values are not set safely: they may be incorrect
 712 +        or even refer to a line that is invalid for the file name. */
 713 +       lock->last_s_file_name = file_name;
 714 +       lock->last_s_line = line;
 715
 716 -       return(FALSE);  /* locking did not succeed */
 717 +       return(TRUE);   /* locking succeeded */
 718  }
 719
 720  /**********************************************************************
 721 @@ -167,11 +340,10 @@
 722         const char*     file_name,      /* in: file name where requested */
 723         ulint           line)           /* in: line where lock requested */
 724  {
 725 -       ut_ad(lock->writer == RW_LOCK_NOT_LOCKED);
 726 -       ut_ad(rw_lock_get_reader_count(lock) == 0);
 727 +       ut_ad(lock->lock_word == X_LOCK_DECR);
 728
 729 -       /* Set the shared lock by incrementing the reader count */
 730 -       lock->reader_count++;
 731 +       /* Indicate there is a new reader by decrementing lock_word */
 732 +       lock->lock_word--;
 733
 734         lock->last_s_file_name = file_name;
 735         lock->last_s_line = line;
 736 @@ -194,13 +366,11 @@
 737         ulint           line)           /* in: line where lock requested */
 738  {
 739          ut_ad(rw_lock_validate(lock));
 740 -       ut_ad(rw_lock_get_reader_count(lock) == 0);
 741 -       ut_ad(rw_lock_get_writer(lock) == RW_LOCK_NOT_LOCKED);
 742 +       ut_ad(lock->lock_word == X_LOCK_DECR);
 743
 744 -       rw_lock_set_writer(lock, RW_LOCK_EX);
 745 +       lock->lock_word -= X_LOCK_DECR;
 746         lock->writer_thread = os_thread_get_curr_id();
 747 -       lock->writer_count++;
 748 -       lock->pass = 0;
 749 +       lock->recursive = TRUE;
 750
 751         lock->last_x_file_name = file_name;
 752         lock->last_x_line = line;
 753 @@ -241,15 +411,12 @@
 754         ut_ad(!rw_lock_own(lock, RW_LOCK_SHARED)); /* see NOTE above */
 755  #endif /* UNIV_SYNC_DEBUG */
 756
 757 -       mutex_enter(rw_lock_get_mutex(lock));
 758 -
 759 -       if (UNIV_LIKELY(rw_lock_s_lock_low(lock, pass, file_name, line))) {
 760 -               mutex_exit(rw_lock_get_mutex(lock));
 761 +       /* TODO: study performance of UNIV_LIKELY branch prediction hints. */
 762 +       if (rw_lock_s_lock_low(lock, pass, file_name, line)) {
 763
 764                 return; /* Success */
 765         } else {
 766                 /* Did not succeed, try spin wait */
 767 -               mutex_exit(rw_lock_get_mutex(lock));
 768
 769                 rw_lock_s_lock_spin(lock, pass, file_name, line);
 770
 771 @@ -259,86 +426,60 @@
 772
 773  /**********************************************************************
 774  NOTE! Use the corresponding macro, not directly this function! Lock an
 775 -rw-lock in shared mode for the current thread if the lock can be acquired
 776 -immediately. */
 777 +rw-lock in exclusive mode for the current thread if the lock can be
 778 +obtained immediately. */
 779  UNIV_INLINE
 780  ibool
 781 -rw_lock_s_lock_func_nowait(
 782 +rw_lock_x_lock_func_nowait(
 783  /*=======================*/
 784                                 /* out: TRUE if success */
 785          rw_lock_t*     lock,   /* in: pointer to rw-lock */
 786         const char*     file_name,/* in: file name where lock requested */
 787         ulint           line)   /* in: line where requested */
 788  {
 789 -       ibool   success = FALSE;
 790 -
 791 -       mutex_enter(rw_lock_get_mutex(lock));
 792 -
 793 -       if (lock->writer == RW_LOCK_NOT_LOCKED) {
 794 -               /* Set the shared lock by incrementing the reader count */
 795 -               lock->reader_count++;
 796 +       os_thread_id_t  curr_thread     = os_thread_get_curr_id();
 797
 798 -#ifdef UNIV_SYNC_DEBUG
 799 -               rw_lock_add_debug_info(lock, 0, RW_LOCK_SHARED, file_name,
 800 -                                                                       line);
 801 -#endif
 802 +       ibool success;
 803
 804 -               lock->last_s_file_name = file_name;
 805 -               lock->last_s_line = line;
 806 +#ifdef HAVE_ATOMIC_BUILTINS
 807 +       success = os_compare_and_swap(&(lock->lock_word), X_LOCK_DECR, 0);
 808 +#else
 809
 810 +       success = FALSE;
 811 +       mutex_enter(&(lock->mutex));
 812 +       if (lock->lock_word == X_LOCK_DECR) {
 813 +               lock->lock_word = 0;
 814                 success = TRUE;
 815         }
 816 +       mutex_exit(&(lock->mutex));
 817
 818 -       mutex_exit(rw_lock_get_mutex(lock));
 819 -
 820 -       return(success);
 821 -}
 822 +#endif
 823 +       if (success) {
 824 +               rw_lock_set_writer_id_and_recursion_flag(lock, TRUE);
 825
 826 -/**********************************************************************
 827 -NOTE! Use the corresponding macro, not directly this function! Lock an
 828 -rw-lock in exclusive mode for the current thread if the lock can be
 829 -obtained immediately. */
 830 -UNIV_INLINE
 831 -ibool
 832 -rw_lock_x_lock_func_nowait(
 833 -/*=======================*/
 834 -                               /* out: TRUE if success */
 835 -        rw_lock_t*     lock,   /* in: pointer to rw-lock */
 836 -       const char*     file_name,/* in: file name where lock requested */
 837 -       ulint           line)   /* in: line where requested */
 838 -{
 839 -       ibool           success         = FALSE;
 840 -       os_thread_id_t  curr_thread     = os_thread_get_curr_id();
 841 -       mutex_enter(rw_lock_get_mutex(lock));
 842 +       } else if (lock->recursive
 843 +                  && os_thread_eq(lock->writer_thread, curr_thread)) {
 844 +               /* Relock: this lock_word modification is safe since no other
 845 +               threads can modify (lock, unlock, or reserve) lock_word while
 846 +               there is an exclusive writer and this is the writer thread. */
 847 +               lock->lock_word -= X_LOCK_DECR;
 848
 849 -       if (UNIV_UNLIKELY(rw_lock_get_reader_count(lock) != 0)) {
 850 -       } else if (UNIV_LIKELY(rw_lock_get_writer(lock)
 851 -                              == RW_LOCK_NOT_LOCKED)) {
 852 -               rw_lock_set_writer(lock, RW_LOCK_EX);
 853 -               lock->writer_thread = curr_thread;
 854 -               lock->pass = 0;
 855 -       relock:
 856 -               lock->writer_count++;
 857 +               ut_ad(((-lock->lock_word) % X_LOCK_DECR) == 0);
 858
 859 +       } else {
 860 +               /* Failure */
 861 +               return(FALSE);
 862 +       }
 863  #ifdef UNIV_SYNC_DEBUG
 864 -               rw_lock_add_debug_info(lock, 0, RW_LOCK_EX, file_name, line);
 865 +       rw_lock_add_debug_info(lock, 0, RW_LOCK_EX, file_name, line);
 866  #endif
 867
 868 -               lock->last_x_file_name = file_name;
 869 -               lock->last_x_line = line;
 870 -
 871 -               success = TRUE;
 872 -       } else if (rw_lock_get_writer(lock) == RW_LOCK_EX
 873 -                       && lock->pass == 0
 874 -                       && os_thread_eq(lock->writer_thread, curr_thread)) {
 875 -               goto relock;
 876 -       }
 877 -
 878 -       mutex_exit(rw_lock_get_mutex(lock));
 879 +       lock->last_x_file_name = file_name;
 880 +       lock->last_x_line = line;
 881
 882          ut_ad(rw_lock_validate(lock));
 883
 884 -       return(success);
 885 +       return(TRUE);
 886  }
 887
 888  /**********************************************************************
 889 @@ -354,39 +495,21 @@
 890  #endif
 891         )
 892  {
 893 -       mutex_t*        mutex   = &(lock->mutex);
 894 -       ibool           sg      = FALSE;
 895 -
 896 -        /* Acquire the mutex protecting the rw-lock fields */
 897 -       mutex_enter(mutex);
 898 -
 899 -       /* Reset the shared lock by decrementing the reader count */
 900 -
 901 -       ut_a(lock->reader_count > 0);
 902 -       lock->reader_count--;
 903 +       ut_ad((lock->lock_word % X_LOCK_DECR) != 0);
 904
 905  #ifdef UNIV_SYNC_DEBUG
 906         rw_lock_remove_debug_info(lock, pass, RW_LOCK_SHARED);
 907  #endif
 908
 909 -       /* If there may be waiters and this was the last s-lock,
 910 -       signal the object */
 911 +       /* Increment lock_word to indicate 1 less reader */
 912 +       if (rw_lock_lock_word_incr(lock, 1) == 0) {
 913
 914 -       if (UNIV_UNLIKELY(lock->waiters)
 915 -                       && lock->reader_count == 0) {
 916 -               sg = TRUE;
 917 -
 918 -               rw_lock_set_waiters(lock, 0);
 919 -       }
 920 -
 921 -       mutex_exit(mutex);
 922 -
 923 -       if (UNIV_UNLIKELY(sg)) {
 924 -#ifdef __WIN__
 925 +               /* wait_ex waiter exists. It may not be asleep, but we signal
 926 +                anyway. We do not wake other waiters, because they can't
 927 +                exist without wait_ex waiter and wait_ex waiter goes first.*/
 928                 os_event_set(lock->wait_ex_event);
 929 -#endif
 930 -               os_event_set(lock->event);
 931                 sync_array_object_signalled(sync_primary_wait_array);
 932 +
 933         }
 934
 935          ut_ad(rw_lock_validate(lock));
 936 @@ -405,16 +528,15 @@
 937  /*====================*/
 938         rw_lock_t*      lock)   /* in: rw-lock */
 939  {
 940 -       /* Reset the shared lock by decrementing the reader count */
 941 -
 942 -       ut_ad(lock->reader_count > 0);
 943 -
 944 -       lock->reader_count--;
 945 +       ut_ad(lock->lock_word < X_LOCK_DECR);
 946
 947  #ifdef UNIV_SYNC_DEBUG
 948         rw_lock_remove_debug_info(lock, 0, RW_LOCK_SHARED);
 949  #endif
 950
 951 +       /* Decrease reader count by incrementing lock_word */
 952 +       lock->lock_word++;
 953 +
 954         ut_ad(!lock->waiters);
 955          ut_ad(rw_lock_validate(lock));
 956  #ifdef UNIV_SYNC_PERF_STAT
 957 @@ -435,42 +557,32 @@
 958  #endif
 959         )
 960  {
 961 -       ibool   sg      = FALSE;
 962 -
 963 -        /* Acquire the mutex protecting the rw-lock fields */
 964 -       mutex_enter(&(lock->mutex));
 965 -
 966 -       /* Reset the exclusive lock if this thread no longer has an x-mode
 967 -       lock */
 968 -
 969 -       ut_ad(lock->writer_count > 0);
 970 +       ut_ad((lock->lock_word % X_LOCK_DECR) == 0);
 971
 972 -       lock->writer_count--;
 973 -
 974 -       if (lock->writer_count == 0) {
 975 -               rw_lock_set_writer(lock, RW_LOCK_NOT_LOCKED);
 976 +       /* lock->recursive flag also indicates if lock->writer_thread is
 977 +       valid or stale. If we are the last of the recursive callers
 978 +       then we must unset lock->recursive flag to indicate that the
 979 +       lock->writer_thread is now stale.
 980 +       Note that since we still hold the x-lock we can safely read the
 981 +       lock_word. */
 982 +       if (lock->lock_word == 0) {
 983 +               /* Last caller in a possible recursive chain. */
 984 +               lock->recursive = FALSE;
 985         }
 986
 987  #ifdef UNIV_SYNC_DEBUG
 988         rw_lock_remove_debug_info(lock, pass, RW_LOCK_EX);
 989  #endif
 990
 991 -       /* If there may be waiters, signal the lock */
 992 -       if (UNIV_UNLIKELY(lock->waiters)
 993 -                       && lock->writer_count == 0) {
 994 -
 995 -               sg = TRUE;
 996 -               rw_lock_set_waiters(lock, 0);
 997 -       }
 998 -
 999 -       mutex_exit(&(lock->mutex));
1000 -
1001 -       if (UNIV_UNLIKELY(sg)) {
1002 -#ifdef __WIN__
1003 -               os_event_set(lock->wait_ex_event);
1004 -#endif
1005 -               os_event_set(lock->event);
1006 -               sync_array_object_signalled(sync_primary_wait_array);
1007 +       if (rw_lock_lock_word_incr(lock, X_LOCK_DECR) == X_LOCK_DECR) {
1008 +               /* Lock is now free. May have to signal read/write waiters.
1009 +                We do not need to signal wait_ex waiters, since they cannot
1010 +                exist when there is a writer. */
1011 +               if (lock->waiters) {
1012 +                       rw_lock_reset_waiter_flag(lock);
1013 +                       os_event_set(lock->event);
1014 +                       sync_array_object_signalled(sync_primary_wait_array);
1015 +               }
1016         }
1017
1018          ut_ad(rw_lock_validate(lock));
1019 @@ -492,18 +604,18 @@
1020         /* Reset the exclusive lock if this thread no longer has an x-mode
1021         lock */
1022
1023 -       ut_ad(lock->writer_count > 0);
1024 -
1025 -       lock->writer_count--;
1026 -
1027 -       if (lock->writer_count == 0) {
1028 -               rw_lock_set_writer(lock, RW_LOCK_NOT_LOCKED);
1029 -       }
1030 +       ut_ad((lock->lock_word % X_LOCK_DECR) == 0);
1031
1032  #ifdef UNIV_SYNC_DEBUG
1033         rw_lock_remove_debug_info(lock, 0, RW_LOCK_EX);
1034  #endif
1035
1036 +       if (lock->lock_word == 0) {
1037 +               lock->recursive = FALSE;
1038 +       }
1039 +
1040 +       lock->lock_word += X_LOCK_DECR;
1041 +
1042         ut_ad(!lock->waiters);
1043          ut_ad(rw_lock_validate(lock));
1044
1045 diff -ruN a/innobase/include/sync0sync.h b/innobase/include/sync0sync.h
1046 --- a/innobase/include/sync0sync.h      2009-10-22 15:15:05.000000000 +0900
1047 +++ b/innobase/include/sync0sync.h      2009-10-22 15:18:44.000000000 +0900
1048 @@ -1,8 +1,31 @@
1049 +/*****************************************************************************
1050 +
1051 +Copyright (c) 1995, 2009, Innobase Oy. All Rights Reserved.
1052 +Copyright (c) 2008, Google Inc.
1053 +
1054 +Portions of this file contain modifications contributed and copyrighted by
1055 +Google, Inc. Those modifications are gratefully acknowledged and are described
1056 +briefly in the InnoDB documentation. The contributions by Google are
1057 +incorporated with their permission, and subject to the conditions contained in
1058 +the file COPYING.Google.
1059 +
1060 +This program is free software; you can redistribute it and/or modify it under
1061 +the terms of the GNU General Public License as published by the Free Software
1062 +Foundation; version 2 of the License.
1063 +
1064 +This program is distributed in the hope that it will be useful, but WITHOUT
1065 +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
1066 +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
1067 +
1068 +You should have received a copy of the GNU General Public License along with
1069 +this program; if not, write to the Free Software Foundation, Inc., 59 Temple
1070 +Place, Suite 330, Boston, MA 02111-1307 USA
1071 +
1072 +*****************************************************************************/
1073 +
1074  /******************************************************
1075  Mutex, the basic synchronization primitive
1076
1077 -(c) 1995 Innobase Oy
1078 -
1079  Created 9/5/1995 Heikki Tuuri
1080  *******************************************************/
1081
1082 @@ -465,8 +488,11 @@
1083  struct mutex_struct {
1084         os_event_t      event;  /* Used by sync0arr.c for the wait queue */
1085         ulint   lock_word;      /* This ulint is the target of the atomic
1086 -                               test-and-set instruction in Win32 */
1087 -#if !defined(_WIN32) || !defined(UNIV_CAN_USE_X86_ASSEMBLER)
1088 +                               test-and-set instruction in Win32 and
1089 +                               x86 32/64 with GCC 4.1.0 or later version */
1090 +#if defined(_WIN32) && defined(UNIV_CAN_USE_X86_ASSEMBLER)
1091 +#elif defined(HAVE_ATOMIC_BUILTINS)
1092 +#else
1093         os_fast_mutex_t
1094                 os_fast_mutex;  /* In other systems we use this OS mutex
1095                                 in place of lock_word */
1096 @@ -525,8 +551,7 @@
1097  /* The number of system calls made in this module. Intended for performance
1098  monitoring. */
1099
1100 -extern         ulint   mutex_system_call_count;
1101 -extern ulint   mutex_exit_count;
1102 +extern ib_longlong     mutex_exit_count;
1103
1104  /* Latching order checks start when this is set TRUE */
1105  extern ibool   sync_order_checks_on;
1106 diff -ruN a/innobase/include/sync0sync.ic b/innobase/include/sync0sync.ic
1107 --- a/innobase/include/sync0sync.ic     2009-09-10 04:02:59.000000000 +0900
1108 +++ b/innobase/include/sync0sync.ic     2009-10-22 15:18:44.000000000 +0900
1109 @@ -1,21 +1,34 @@
1110 +/*****************************************************************************
1111 +
1112 +Copyright (c) 1995, 2009, Innobase Oy. All Rights Reserved.
1113 +Copyright (c) 2008, Google Inc.
1114 +
1115 +Portions of this file contain modifications contributed and copyrighted by
1116 +Google, Inc. Those modifications are gratefully acknowledged and are described
1117 +briefly in the InnoDB documentation. The contributions by Google are
1118 +incorporated with their permission, and subject to the conditions contained in
1119 +the file COPYING.Google.
1120 +
1121 +This program is free software; you can redistribute it and/or modify it under
1122 +the terms of the GNU General Public License as published by the Free Software
1123 +Foundation; version 2 of the License.
1124 +
1125 +This program is distributed in the hope that it will be useful, but WITHOUT
1126 +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
1127 +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
1128 +
1129 +You should have received a copy of the GNU General Public License along with
1130 +this program; if not, write to the Free Software Foundation, Inc., 59 Temple
1131 +Place, Suite 330, Boston, MA 02111-1307 USA
1132 +
1133 +*****************************************************************************/
1134 +
1135  /******************************************************
1136  Mutex, the basic synchronization primitive
1137
1138 -(c) 1995 Innobase Oy
1139 -
1140  Created 9/5/1995 Heikki Tuuri
1141  *******************************************************/
1142
1143 -#if defined(not_defined) && defined(__GNUC__) && defined(UNIV_INTEL_X86)
1144 -/* %z0: Use the size of operand %0 which in our case is *m to determine
1145 -instruction size, it should end up as xchgl. "1" in the input constraint,
1146 -says that "in" has to go in the same place as "out".*/
1147 -#define TAS(m, in, out) \
1148 -       asm volatile ("xchg%z0 %2, %0" \
1149 -       : "=g" (*(m)), "=r" (out) \
1150 -       : "1" (in))     /* Note: "1" here refers to "=r" (out) */
1151 -#endif
1152 -
1153  /**********************************************************************
1154  Sets the waiters field in a mutex. */
1155
1156 @@ -94,12 +107,8 @@
1157         /* mutex_fence(); */
1158
1159         return(res);
1160 -#elif defined(not_defined) && defined(__GNUC__) && defined(UNIV_INTEL_X86)
1161 -       ulint   res;
1162 -
1163 -       TAS(&mutex->lock_word, 1, res);
1164 -
1165 -       return(res);
1166 +#elif defined(HAVE_ATOMIC_BUILTINS)
1167 +       return __sync_lock_test_and_set(&(mutex->lock_word), 1);
1168  #else
1169         ibool   ret;
1170
1171 @@ -136,10 +145,11 @@
1172         __asm   MOV     EDX, 0
1173          __asm   MOV     ECX, lw
1174          __asm   XCHG    EDX, DWORD PTR [ECX]
1175 -#elif defined(not_defined) && defined(__GNUC__) && defined(UNIV_INTEL_X86)
1176 -       ulint   res;
1177 -
1178 -       TAS(&mutex->lock_word, 0, res);
1179 +#elif defined(HAVE_ATOMIC_BUILTINS)
1180 +       /* In theory __sync_lock_release should be used to release the lock.
1181 +       Unfortunately, it does not work properly alone. The workaround is
1182 +       that more conservative __sync_lock_test_and_set is used instead. */
1183 +       __sync_lock_test_and_set(&(mutex->lock_word), 0);
1184  #else
1185         mutex->lock_word = 0;
1186
1187 diff -ruN a/innobase/row/row0sel.c b/innobase/row/row0sel.c
1188 --- a/innobase/row/row0sel.c    2009-10-22 15:15:05.000000000 +0900
1189 +++ b/innobase/row/row0sel.c    2009-10-22 15:18:44.000000000 +0900
1190 @@ -1178,7 +1178,7 @@
1191                         rw_lock_s_lock(&btr_search_latch);
1192
1193                         search_latch_locked = TRUE;
1194 -               } else if (btr_search_latch.writer_is_wait_ex) {
1195 +               } else if (rw_lock_get_writer(&btr_search_latch) == RW_LOCK_WAIT_EX) {
1196
1197                         /* There is an x-latch request waiting: release the
1198                         s-latch for a moment; as an s-latch here is often
1199 @@ -3123,7 +3123,7 @@
1200         /* PHASE 0: Release a possible s-latch we are holding on the
1201         adaptive hash index latch if there is someone waiting behind */
1202
1203 -       if (UNIV_UNLIKELY(btr_search_latch.writer != RW_LOCK_NOT_LOCKED)
1204 +       if (UNIV_UNLIKELY(rw_lock_get_writer(&btr_search_latch) != RW_LOCK_NOT_LOCKED)
1205             && trx->has_search_latch) {
1206
1207                 /* There is an x-latch request on the adaptive hash index:
1208 diff -ruN a/innobase/sync/sync0arr.c b/innobase/sync/sync0arr.c
1209 --- a/innobase/sync/sync0arr.c  2009-09-10 04:03:01.000000000 +0900
1210 +++ b/innobase/sync/sync0arr.c  2009-10-22 15:18:44.000000000 +0900
1211 @@ -1,8 +1,31 @@
1212 +/*****************************************************************************
1213 +
1214 +Copyright (c) 1995, 2009, Innobase Oy. All Rights Reserved.
1215 +Copyright (c) 2008, Google Inc.
1216 +
1217 +Portions of this file contain modifications contributed and copyrighted by
1218 +Google, Inc. Those modifications are gratefully acknowledged and are described
1219 +briefly in the InnoDB documentation. The contributions by Google are
1220 +incorporated with their permission, and subject to the conditions contained in
1221 +the file COPYING.Google.
1222 +
1223 +This program is free software; you can redistribute it and/or modify it under
1224 +the terms of the GNU General Public License as published by the Free Software
1225 +Foundation; version 2 of the License.
1226 +
1227 +This program is distributed in the hope that it will be useful, but WITHOUT
1228 +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
1229 +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
1230 +
1231 +You should have received a copy of the GNU General Public License along with
1232 +this program; if not, write to the Free Software Foundation, Inc., 59 Temple
1233 +Place, Suite 330, Boston, MA 02111-1307 USA
1234 +
1235 +*****************************************************************************/
1236 +
1237  /******************************************************
1238  The wait array used in synchronization primitives
1239
1240 -(c) 1995 Innobase Oy
1241 -
1242  Created 9/5/1995 Heikki Tuuri
1243  *******************************************************/
1244
1245 @@ -297,25 +320,21 @@
1246  }
1247
1248  /***********************************************************************
1249 -Puts the cell event in reset state. */
1250 +Returns the event that the thread owning the cell waits for. */
1251  static
1252 -ib_longlong
1253 -sync_cell_event_reset(
1254 -/*==================*/
1255 -                               /* out: value of signal_count
1256 -                               at the time of reset. */
1257 -       ulint           type,   /* in: lock type mutex/rw_lock */
1258 -       void*           object) /* in: the rw_lock/mutex object */
1259 +os_event_t
1260 +sync_cell_get_event(
1261 +/*================*/
1262 +       sync_cell_t*    cell) /* in: non-empty sync array cell */
1263  {
1264 +       ulint type = cell->request_type;
1265 +
1266         if (type == SYNC_MUTEX) {
1267 -               return(os_event_reset(((mutex_t *) object)->event));
1268 -#ifdef __WIN__
1269 +               return(((mutex_t *) cell->wait_object)->event);
1270         } else if (type == RW_LOCK_WAIT_EX) {
1271 -               return(os_event_reset(
1272 -                      ((rw_lock_t *) object)->wait_ex_event));
1273 -#endif
1274 -       } else {
1275 -               return(os_event_reset(((rw_lock_t *) object)->event));
1276 +               return(((rw_lock_t *) cell->wait_object)->wait_ex_event);
1277 +       } else { /* RW_LOCK_SHARED and RW_LOCK_EX wait on the same event */
1278 +               return(((rw_lock_t *) cell->wait_object)->event);
1279         }
1280  }
1281
1282 @@ -334,6 +353,7 @@
1283          ulint*         index)  /* out: index of the reserved cell */
1284  {
1285          sync_cell_t*           cell;
1286 +       os_event_t      event;
1287          ulint           i;
1288
1289          ut_a(object);
1290 @@ -372,8 +392,8 @@
1291                         /* Make sure the event is reset and also store
1292                         the value of signal_count at which the event
1293                         was reset. */
1294 -                       cell->signal_count = sync_cell_event_reset(type,
1295 -                                                               object);
1296 +                        event = sync_cell_get_event(cell);
1297 +                       cell->signal_count = os_event_reset(event);
1298
1299                         cell->reservation_time = time(NULL);
1300
1301 @@ -413,19 +433,7 @@
1302         ut_a(!cell->waiting);
1303         ut_ad(os_thread_get_curr_id() == cell->thread);
1304
1305 -       if (cell->request_type == SYNC_MUTEX) {
1306 -               event = ((mutex_t*) cell->wait_object)->event;
1307 -#ifdef __WIN__
1308 -       /* On windows if the thread about to wait is the one which
1309 -       has set the state of the rw_lock to RW_LOCK_WAIT_EX, then
1310 -       it waits on a special event i.e.: wait_ex_event. */
1311 -       } else if (cell->request_type == RW_LOCK_WAIT_EX) {
1312 -               event = ((rw_lock_t*) cell->wait_object)->wait_ex_event;
1313 -#endif
1314 -       } else {
1315 -               event = ((rw_lock_t*) cell->wait_object)->event;
1316 -       }
1317 -
1318 +       event = sync_cell_get_event(cell);
1319                 cell->waiting = TRUE;
1320
1321  #ifdef UNIV_SYNC_DEBUG
1322 @@ -464,6 +472,7 @@
1323         mutex_t*        mutex;
1324         rw_lock_t*      rwlock;
1325         ulint           type;
1326 +       ulint           writer;
1327
1328         type = cell->request_type;
1329
1330 @@ -492,9 +501,7 @@
1331                         (ulong) mutex->waiters);
1332
1333         } else if (type == RW_LOCK_EX
1334 -#ifdef __WIN__
1335                    || type == RW_LOCK_WAIT_EX
1336 -#endif
1337                    || type == RW_LOCK_SHARED) {
1338
1339                 fputs(type == RW_LOCK_EX ? "X-lock on" : "S-lock on", file);
1340 @@ -505,21 +512,24 @@
1341                         " RW-latch at %p created in file %s line %lu\n",
1342                         rwlock, rwlock->cfile_name,
1343                         (ulong) rwlock->cline);
1344 -               if (rwlock->writer != RW_LOCK_NOT_LOCKED) {
1345 +               writer = rw_lock_get_writer(rwlock);
1346 +               if (writer != RW_LOCK_NOT_LOCKED) {
1347                         fprintf(file,
1348                         "a writer (thread id %lu) has reserved it in mode %s",
1349                                 (ulong) os_thread_pf(rwlock->writer_thread),
1350 -                               rwlock->writer == RW_LOCK_EX
1351 +                               writer == RW_LOCK_EX
1352                                 ? " exclusive\n"
1353                                 : " wait exclusive\n");
1354                 }
1355
1356                 fprintf(file,
1357 -                       "number of readers %lu, waiters flag %lu\n"
1358 +                       "number of readers %lu, waiters flag %lu, "
1359 +                        "lock_word: %lx\n"
1360                         "Last time read locked in file %s line %lu\n"
1361                         "Last time write locked in file %s line %lu\n",
1362 -                       (ulong) rwlock->reader_count,
1363 +                       (ulong) rw_lock_get_reader_count(rwlock),
1364                         (ulong) rwlock->waiters,
1365 +                       rwlock->lock_word,
1366                         rwlock->last_s_file_name,
1367                         (ulong) rwlock->last_s_line,
1368                         rwlock->last_x_file_name,
1369 @@ -773,28 +783,30 @@
1370                         return(TRUE);
1371                 }
1372
1373 -       } else if (cell->request_type == RW_LOCK_EX
1374 -                  || cell->request_type == RW_LOCK_WAIT_EX) {
1375 +       } else if (cell->request_type == RW_LOCK_EX) {
1376
1377                 lock = cell->wait_object;
1378
1379 -               if (rw_lock_get_reader_count(lock) == 0
1380 -                   && rw_lock_get_writer(lock) == RW_LOCK_NOT_LOCKED) {
1381 +               if (lock->lock_word > 0) {
1382 +               /* Either unlocked or only read locked. */
1383
1384                         return(TRUE);
1385                 }
1386
1387 -               if (rw_lock_get_reader_count(lock) == 0
1388 -                   && rw_lock_get_writer(lock) == RW_LOCK_WAIT_EX
1389 -                   && os_thread_eq(lock->writer_thread, cell->thread)) {
1390 +        } else if (cell->request_type == RW_LOCK_WAIT_EX) {
1391 +
1392 +               lock = cell->wait_object;
1393 +
1394 +                /* lock_word == 0 means all readers have left */
1395 +               if (lock->lock_word == 0) {
1396
1397                         return(TRUE);
1398                 }
1399 -
1400         } else if (cell->request_type == RW_LOCK_SHARED) {
1401                 lock = cell->wait_object;
1402
1403 -               if (rw_lock_get_writer(lock) == RW_LOCK_NOT_LOCKED) {
1404 +                /* lock_word > 0 means no writer or reserved writer */
1405 +               if (lock->lock_word > 0) {
1406
1407                         return(TRUE);
1408                 }
1409 @@ -839,11 +851,15 @@
1410  /*========================*/
1411         sync_array_t*   arr)    /* in: wait array */
1412  {
1413 +#ifdef HAVE_ATOMIC_BUILTINS
1414 +       (void) os_atomic_increment(&arr->sg_count, 1);
1415 +#else
1416          sync_array_enter(arr);
1417
1418         arr->sg_count++;
1419
1420          sync_array_exit(arr);
1421 +#endif
1422  }
1423
1424  /**************************************************************************
1425 @@ -859,6 +875,7 @@
1426          sync_cell_t*           cell;
1427          ulint           count;
1428          ulint           i;
1429 +       os_event_t      event;
1430
1431          sync_array_enter(arr);
1432
1433 @@ -868,36 +885,20 @@
1434          while (count < arr->n_reserved) {
1435
1436                 cell = sync_array_get_nth_cell(arr, i);
1437 +               i++;
1438
1439 -                if (cell->wait_object != NULL) {
1440 -
1441 +               if (cell->wait_object == NULL) {
1442 +                       continue;
1443 +               }
1444                          count++;
1445
1446                          if (sync_arr_cell_can_wake_up(cell)) {
1447
1448 -                               if (cell->request_type == SYNC_MUTEX) {
1449 -                                       mutex_t*        mutex;
1450 +                       event = sync_cell_get_event(cell);
1451
1452 -                                       mutex = cell->wait_object;
1453 -                                       os_event_set(mutex->event);
1454 -#ifdef __WIN__
1455 -                               } else if (cell->request_type
1456 -                                          == RW_LOCK_WAIT_EX) {
1457 -                                       rw_lock_t*      lock;
1458 -
1459 -                                       lock = cell->wait_object;
1460 -                                       os_event_set(lock->wait_ex_event);
1461 -#endif
1462 -                               } else {
1463 -                                       rw_lock_t*      lock;
1464 -
1465 -                                       lock = cell->wait_object;
1466 -                                       os_event_set(lock->event);
1467 -                               }
1468 -                        }
1469 +                       os_event_set(event);
1470                  }
1471
1472 -                i++;
1473          }
1474
1475          sync_array_exit(arr);
1476 @@ -1014,4 +1015,3 @@
1477
1478          sync_array_exit(arr);
1479  }
1480 -
1481 diff -ruN a/innobase/sync/sync0rw.c b/innobase/sync/sync0rw.c
1482 --- a/innobase/sync/sync0rw.c   2009-09-10 04:03:01.000000000 +0900
1483 +++ b/innobase/sync/sync0rw.c   2009-10-22 15:18:44.000000000 +0900
1484 @@ -1,8 +1,31 @@
1485 +/*****************************************************************************
1486 +
1487 +Copyright (c) 1995, 2009, Innobase Oy. All Rights Reserved.
1488 +Copyright (c) 2008, Google Inc.
1489 +
1490 +Portions of this file contain modifications contributed and copyrighted by
1491 +Google, Inc. Those modifications are gratefully acknowledged and are described
1492 +briefly in the InnoDB documentation. The contributions by Google are
1493 +incorporated with their permission, and subject to the conditions contained in
1494 +the file COPYING.Google.
1495 +
1496 +This program is free software; you can redistribute it and/or modify it under
1497 +the terms of the GNU General Public License as published by the Free Software
1498 +Foundation; version 2 of the License.
1499 +
1500 +This program is distributed in the hope that it will be useful, but WITHOUT
1501 +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
1502 +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
1503 +
1504 +You should have received a copy of the GNU General Public License along with
1505 +this program; if not, write to the Free Software Foundation, Inc., 59 Temple
1506 +Place, Suite 330, Boston, MA 02111-1307 USA
1507 +
1508 +*****************************************************************************/
1509 +
1510  /******************************************************
1511  The read-write lock (for thread synchronization)
1512
1513 -(c) 1995 Innobase Oy
1514 -
1515  Created 9/11/1995 Heikki Tuuri
1516  *******************************************************/
1517
1518 @@ -15,17 +38,110 @@
1519  #include "mem0mem.h"
1520  #include "srv0srv.h"
1521
1522 -ulint  rw_s_system_call_count  = 0;
1523 -ulint  rw_s_spin_wait_count    = 0;
1524 -ulint  rw_s_os_wait_count      = 0;
1525 +/*
1526 +       IMPLEMENTATION OF THE RW_LOCK
1527 +       =============================
1528 +The status of a rw_lock is held in lock_word. The initial value of lock_word is
1529 +X_LOCK_DECR. lock_word is decremented by 1 for each s-lock and by X_LOCK_DECR
1530 +for each x-lock. This describes the lock state for each value of lock_word:
1531 +
1532 +lock_word == X_LOCK_DECR:      Unlocked.
1533 +0 < lock_word < X_LOCK_DECR:   Read locked, no waiting writers.
1534 +                              (X_LOCK_DECR - lock_word) is the
1535 +                              number of readers that hold the lock.
1536 +lock_word == 0:                       Write locked
1537 +-X_LOCK_DECR < lock_word < 0:  Read locked, with a waiting writer.
1538 +                              (-lock_word) is the number of readers
1539 +                              that hold the lock.
1540 +lock_word <= -X_LOCK_DECR:     Recursively write locked. lock_word has been
1541 +                              decremented by X_LOCK_DECR once for each lock,
1542 +                              so the number of locks is:
1543 +                              ((-lock_word) / X_LOCK_DECR) + 1
1544 +When lock_word <= -X_LOCK_DECR, we also know that lock_word % X_LOCK_DECR == 0:
1545 +other values of lock_word are invalid.
1546 +
1547 +The lock_word is always read and updated atomically and consistently, so that
1548 +it always represents the state of the lock, and the state of the lock changes
1549 +with a single atomic operation. This lock_word holds all of the information
1550 +that a thread needs in order to determine if it is eligible to gain the lock
1551 +or if it must spin or sleep. The one exception to this is that writer_thread
1552 +must be verified before recursive write locks: to solve this scenario, we make
1553 +writer_thread readable by all threads, but only writeable by the x-lock holder.
1554 +
1555 +The other members of the lock obey the following rules to remain consistent:
1556 +
1557 +recursive:     This and the writer_thread field together control the
1558 +               behaviour of recursive x-locking.
1559 +               lock->recursive must be FALSE in following states:
1560 +                       1) The writer_thread contains garbage i.e.: the
1561 +                       lock has just been initialized.
1562 +                       2) The lock is not x-held and there is no
1563 +                       x-waiter waiting on WAIT_EX event.
1564 +                       3) The lock is x-held or there is an x-waiter
1565 +                       waiting on WAIT_EX event but the 'pass' value
1566 +                       is non-zero.
1567 +               lock->recursive is TRUE iff:
1568 +                       1) The lock is x-held or there is an x-waiter
1569 +                       waiting on WAIT_EX event and the 'pass' value
1570 +                       is zero.
1571 +               This flag must be set after the writer_thread field
1572 +               has been updated with a memory ordering barrier.
1573 +               It is unset before the lock_word has been incremented.
1574 +writer_thread: Is used only in recursive x-locking. Can only be safely
1575 +               read iff lock->recursive flag is TRUE.
1576 +               This field is uninitialized at lock creation time and
1577 +               is updated atomically when x-lock is acquired or when
1578 +               move_ownership is called. A thread is only allowed to
1579 +               set the value of this field to it's thread_id i.e.: a
1580 +               thread cannot set writer_thread to some other thread's
1581 +               id.
1582 +waiters:       May be set to 1 anytime, but to avoid unnecessary wake-up
1583 +               signals, it should only be set to 1 when there are threads
1584 +               waiting on event. Must be 1 when a writer starts waiting to
1585 +               ensure the current x-locking thread sends a wake-up signal
1586 +               during unlock. May only be reset to 0 immediately before a
1587 +               a wake-up signal is sent to event. On most platforms, a
1588 +               memory barrier is required after waiters is set, and before
1589 +               verifying lock_word is still held, to ensure some unlocker
1590 +               really does see the flags new value.
1591 +event:         Threads wait on event for read or writer lock when another
1592 +               thread has an x-lock or an x-lock reservation (wait_ex). A
1593 +               thread may only wait on event after performing the following
1594 +               actions in order:
1595 +                  (1) Record the counter value of event (with os_event_reset).
1596 +                  (2) Set waiters to 1.
1597 +                  (3) Verify lock_word <= 0.
1598 +               (1) must come before (2) to ensure signal is not missed.
1599 +               (2) must come before (3) to ensure a signal is sent.
1600 +               These restrictions force the above ordering.
1601 +               Immediately before sending the wake-up signal, we should:
1602 +                  (1) Verify lock_word == X_LOCK_DECR (unlocked)
1603 +                  (2) Reset waiters to 0.
1604 +wait_ex_event: A thread may only wait on the wait_ex_event after it has
1605 +               performed the following actions in order:
1606 +                  (1) Decrement lock_word by X_LOCK_DECR.
1607 +                  (2) Record counter value of wait_ex_event (os_event_reset,
1608 +                       called from sync_array_reserve_cell).
1609 +                  (3) Verify that lock_word < 0.
1610 +               (1) must come first to ensures no other threads become reader
1611 +                or next writer, and notifies unlocker that signal must be sent.
1612 +                (2) must come before (3) to ensure the signal is not missed.
1613 +               These restrictions force the above ordering.
1614 +               Immediately before sending the wake-up signal, we should:
1615 +                  Verify lock_word == 0 (waiting thread holds x_lock)
1616 +*/
1617 +
1618 +ib_longlong    rw_s_spin_wait_count    = 0;
1619 +ib_longlong    rw_s_spin_round_count   = 0;
1620 +ib_longlong    rw_s_os_wait_count      = 0;
1621 +
1622 +ib_longlong    rw_s_exit_count         = 0;
1623 +
1624 +ib_longlong    rw_x_spin_wait_count    = 0;
1625 +ib_longlong    rw_x_spin_round_count   = 0;
1626 +ib_longlong    rw_x_os_wait_count      = 0;
1627
1628 -ulint  rw_s_exit_count         = 0;
1629 -
1630 -ulint  rw_x_system_call_count  = 0;
1631 -ulint  rw_x_spin_wait_count    = 0;
1632 -ulint  rw_x_os_wait_count      = 0;
1633 -
1634 -ulint  rw_x_exit_count         = 0;
1635 +ib_longlong    rw_x_exit_count         = 0;
1636
1637  /* The global list of rw-locks */
1638  rw_lock_list_t rw_lock_list;
1639 @@ -99,22 +215,30 @@
1640         object is created, then the following call initializes
1641         the sync system. */
1642
1643 +#ifndef HAVE_ATOMIC_BUILTINS
1644         mutex_create(rw_lock_get_mutex(lock));
1645         mutex_set_level(rw_lock_get_mutex(lock), SYNC_NO_ORDER_CHECK);
1646
1647         lock->mutex.cfile_name = cfile_name;
1648         lock->mutex.cline = cline;
1649 -#if defined UNIV_DEBUG && !defined UNIV_HOTBACKUP
1650 +# if defined UNIV_DEBUG && !defined UNIV_HOTBACKUP
1651         lock->mutex.cmutex_name = cmutex_name;
1652         lock->mutex.mutex_type = 1;
1653 -#endif /* UNIV_DEBUG && !UNIV_HOTBACKUP */
1654 +# endif /* UNIV_DEBUG && !UNIV_HOTBACKUP */
1655
1656 -       rw_lock_set_waiters(lock, 0);
1657 -       rw_lock_set_writer(lock, RW_LOCK_NOT_LOCKED);
1658 -       lock->writer_count = 0;
1659 -       rw_lock_set_reader_count(lock, 0);
1660 -
1661 -       lock->writer_is_wait_ex = FALSE;
1662 +#else /* HAVE_ATOMIC_BUILTINS */
1663 +# ifdef UNIV_DEBUG
1664 +       UT_NOT_USED(cmutex_name);
1665 +# endif
1666 +#endif /* HAVE_ATOMIC_BUILTINS */
1667 +
1668 +       lock->lock_word = X_LOCK_DECR;
1669 +       lock->waiters = 0;
1670 +
1671 +       /* We set this value to signify that lock->writer_thread
1672 +       contains garbage at initialization and cannot be used for
1673 +       recursive x-locking. */
1674 +       lock->recursive = FALSE;
1675
1676  #ifdef UNIV_SYNC_DEBUG
1677         UT_LIST_INIT(lock->debug_list);
1678 @@ -126,15 +250,13 @@
1679         lock->cfile_name = cfile_name;
1680         lock->cline = cline;
1681
1682 +       lock->count_os_wait = 0;
1683         lock->last_s_file_name = "not yet reserved";
1684         lock->last_x_file_name = "not yet reserved";
1685         lock->last_s_line = 0;
1686         lock->last_x_line = 0;
1687         lock->event = os_event_create(NULL);
1688 -
1689 -#ifdef __WIN__
1690         lock->wait_ex_event = os_event_create(NULL);
1691 -#endif
1692
1693         mutex_enter(&rw_lock_list_mutex);
1694
1695 @@ -158,23 +280,17 @@
1696  /*=========*/
1697         rw_lock_t*      lock)   /* in: rw-lock */
1698  {
1699 -#ifdef UNIV_DEBUG
1700         ut_a(rw_lock_validate(lock));
1701 -#endif /* UNIV_DEBUG */
1702 -       ut_a(rw_lock_get_writer(lock) == RW_LOCK_NOT_LOCKED);
1703 -       ut_a(rw_lock_get_waiters(lock) == 0);
1704 -       ut_a(rw_lock_get_reader_count(lock) == 0);
1705 +       ut_a(lock->lock_word == X_LOCK_DECR);
1706
1707 -       lock->magic_n = 0;
1708 -
1709 +#ifndef HAVE_ATOMIC_BUILTINS
1710         mutex_free(rw_lock_get_mutex(lock));
1711 +#endif /* HAVE_ATOMIC_BUILTINS */
1712
1713         mutex_enter(&rw_lock_list_mutex);
1714         os_event_free(lock->event);
1715
1716 -#ifdef __WIN__
1717         os_event_free(lock->wait_ex_event);
1718 -#endif
1719
1720         if (UT_LIST_GET_PREV(list, lock)) {
1721                 ut_a(UT_LIST_GET_PREV(list, lock)->magic_n == RW_LOCK_MAGIC_N);
1722 @@ -186,6 +302,8 @@
1723         UT_LIST_REMOVE(list, rw_lock_list, lock);
1724
1725         mutex_exit(&rw_lock_list_mutex);
1726 +
1727 +       lock->magic_n = 0;
1728  }
1729
1730  /**********************************************************************
1731 @@ -199,19 +317,12 @@
1732  {
1733         ut_a(lock);
1734
1735 -       mutex_enter(rw_lock_get_mutex(lock));
1736 +       ulint waiters = rw_lock_get_waiters(lock);
1737 +       lint lock_word = lock->lock_word;
1738
1739         ut_a(lock->magic_n == RW_LOCK_MAGIC_N);
1740 -       ut_a((rw_lock_get_reader_count(lock) == 0)
1741 -            || (rw_lock_get_writer(lock) != RW_LOCK_EX));
1742 -       ut_a((rw_lock_get_writer(lock) == RW_LOCK_EX)
1743 -            || (rw_lock_get_writer(lock) == RW_LOCK_WAIT_EX)
1744 -            || (rw_lock_get_writer(lock) == RW_LOCK_NOT_LOCKED));
1745 -       ut_a((rw_lock_get_waiters(lock) == 0)
1746 -            || (rw_lock_get_waiters(lock) == 1));
1747 -       ut_a((lock->writer != RW_LOCK_EX) || (lock->writer_count > 0));
1748 -
1749 -       mutex_exit(rw_lock_get_mutex(lock));
1750 +       ut_a(waiters == 0 || waiters == 1);
1751 +       ut_a(lock_word > -X_LOCK_DECR ||(-lock_word) % X_LOCK_DECR == 0);
1752
1753         return(TRUE);
1754  }
1755 @@ -232,18 +343,15 @@
1756         ulint           line)   /* in: line where requested */
1757  {
1758          ulint    index;        /* index of the reserved wait cell */
1759 -        ulint    i;    /* spin round count */
1760 +       ulint    i = 0; /* spin round count */
1761
1762          ut_ad(rw_lock_validate(lock));
1763
1764 +       rw_s_spin_wait_count++; /* Count calls to this function */
1765  lock_loop:
1766 -       rw_s_spin_wait_count++;
1767
1768         /* Spin waiting for the writer field to become free */
1769 -        i = 0;
1770 -
1771 -        while (rw_lock_get_writer(lock) != RW_LOCK_NOT_LOCKED
1772 -                                               && i < SYNC_SPIN_ROUNDS) {
1773 +       while (i < SYNC_SPIN_ROUNDS && lock->lock_word <= 0) {
1774                 if (srv_spin_wait_delay) {
1775                         ut_delay(ut_rnd_interval(0, srv_spin_wait_delay));
1776                 }
1777 @@ -262,28 +370,32 @@
1778                 lock->cfile_name, (ulong) lock->cline, (ulong) i);
1779         }
1780
1781 -       mutex_enter(rw_lock_get_mutex(lock));
1782 -
1783          /* We try once again to obtain the lock */
1784 -
1785         if (TRUE == rw_lock_s_lock_low(lock, pass, file_name, line)) {
1786 -               mutex_exit(rw_lock_get_mutex(lock));
1787 +               rw_s_spin_round_count += i;
1788
1789                 return; /* Success */
1790         } else {
1791 -               /* If we get here, locking did not succeed, we may
1792 -               suspend the thread to wait in the wait array */
1793
1794 -               rw_s_system_call_count++;
1795 +               if (i < SYNC_SPIN_ROUNDS) {
1796 +                       goto lock_loop;
1797 +               }
1798 +
1799 +               rw_s_spin_round_count += i;
1800
1801                 sync_array_reserve_cell(sync_primary_wait_array,
1802                                 lock, RW_LOCK_SHARED,
1803                                 file_name, line,
1804                                 &index);
1805
1806 -               rw_lock_set_waiters(lock, 1);
1807 -
1808 -               mutex_exit(rw_lock_get_mutex(lock));
1809 +               /* Set waiters before checking lock_word to ensure wake-up
1810 +                signal is sent. This may lead to some unnecessary signals. */
1811 +               rw_lock_set_waiter_flag(lock);
1812 +
1813 +               if (TRUE == rw_lock_s_lock_low(lock, pass, file_name, line)) {
1814 +                       sync_array_free_cell(sync_primary_wait_array, index);
1815 +                       return; /* Success */
1816 +               }
1817
1818                 if (srv_print_latch_waits) {
1819                         fprintf(stderr,
1820 @@ -292,11 +404,13 @@
1821                         lock, lock->cfile_name, (ulong) lock->cline);
1822                 }
1823
1824 -               rw_s_system_call_count++;
1825 +               /* these stats may not be accurate */
1826 +               lock->count_os_wait++;
1827                 rw_s_os_wait_count++;
1828
1829                         sync_array_wait_event(sync_primary_wait_array, index);
1830
1831 +               i = 0;
1832                 goto lock_loop;
1833         }
1834  }
1835 @@ -318,114 +432,130 @@
1836  {
1837         ut_ad(rw_lock_is_locked(lock, RW_LOCK_EX));
1838
1839 -       mutex_enter(&(lock->mutex));
1840 -
1841 -       lock->writer_thread = os_thread_get_curr_id();
1842 -
1843 -       lock->pass = 0;
1844 -
1845 -       mutex_exit(&(lock->mutex));
1846 +       rw_lock_set_writer_id_and_recursion_flag(lock, TRUE);
1847  }
1848
1849  /**********************************************************************
1850 -Low-level function for acquiring an exclusive lock. */
1851 +Function for the next writer to call. Waits for readers to exit.
1852 +The caller must have already decremented lock_word by X_LOCK_DECR.*/
1853  UNIV_INLINE
1854 -ulint
1855 -rw_lock_x_lock_low(
1856 -/*===============*/
1857 -                               /* out: RW_LOCK_NOT_LOCKED if did
1858 -                               not succeed, RW_LOCK_EX if success,
1859 -                               RW_LOCK_WAIT_EX, if got wait reservation */
1860 +void
1861 +rw_lock_x_lock_wait(
1862 +/*================*/
1863          rw_lock_t*     lock,   /* in: pointer to rw-lock */
1864 +#ifdef UNIV_SYNC_DEBUG
1865         ulint           pass,   /* in: pass value; != 0, if the lock will
1866                                 be passed to another thread to unlock */
1867 +#endif
1868         const char*     file_name,/* in: file name where lock requested */
1869         ulint           line)   /* in: line where requested */
1870  {
1871 -#ifdef UNIV_SYNC_DEBUG
1872 -       ut_ad(mutex_own(rw_lock_get_mutex(lock)));
1873 -#endif /* UNIV_SYNC_DEBUG */
1874 -       if (rw_lock_get_writer(lock) == RW_LOCK_NOT_LOCKED) {
1875 +       ulint index;
1876 +       ulint i = 0;
1877
1878 -               if (rw_lock_get_reader_count(lock) == 0) {
1879 +       ut_ad(lock->lock_word <= 0);
1880 +
1881 +       while (lock->lock_word < 0) {
1882 +               if (srv_spin_wait_delay) {
1883 +                       ut_delay(ut_rnd_interval(0, srv_spin_wait_delay));
1884 +               }
1885 +               if(i < SYNC_SPIN_ROUNDS) {
1886 +                       i++;
1887 +                       continue;
1888 +               }
1889
1890 -                       rw_lock_set_writer(lock, RW_LOCK_EX);
1891 -                       lock->writer_thread = os_thread_get_curr_id();
1892 -                       lock->writer_count++;
1893 -                       lock->pass = pass;
1894 +               /* If there is still a reader, then go to sleep.*/
1895 +               rw_x_spin_round_count += i;
1896 +               i = 0;
1897 +               sync_array_reserve_cell(sync_primary_wait_array,
1898 +                                       lock,
1899 +                                       RW_LOCK_WAIT_EX,
1900 +                                       file_name, line,
1901 +                                       &index);
1902 +               /* Check lock_word to ensure wake-up isn't missed.*/
1903 +               if(lock->lock_word < 0) {
1904
1905 +                       /* these stats may not be accurate */
1906 +                       lock->count_os_wait++;
1907 +                       rw_x_os_wait_count++;
1908 +
1909 +                        /* Add debug info as it is needed to detect possible
1910 +                        deadlock. We must add info for WAIT_EX thread for
1911 +                        deadlock detection to work properly. */
1912  #ifdef UNIV_SYNC_DEBUG
1913 -                       rw_lock_add_debug_info(lock, pass, RW_LOCK_EX,
1914 +                       rw_lock_add_debug_info(lock, pass, RW_LOCK_WAIT_EX,
1915                                                         file_name, line);
1916  #endif
1917 -                       lock->last_x_file_name = file_name;
1918 -                       lock->last_x_line = line;
1919 -
1920 -                       /* Locking succeeded, we may return */
1921 -                       return(RW_LOCK_EX);
1922 -               } else {
1923 -                       /* There are readers, we have to wait */
1924 -                       rw_lock_set_writer(lock, RW_LOCK_WAIT_EX);
1925 -                       lock->writer_thread = os_thread_get_curr_id();
1926 -                       lock->pass = pass;
1927 -                       lock->writer_is_wait_ex = TRUE;
1928
1929 +                       sync_array_wait_event(sync_primary_wait_array,
1930 +                                             index);
1931  #ifdef UNIV_SYNC_DEBUG
1932 -                       rw_lock_add_debug_info(lock, pass, RW_LOCK_WAIT_EX,
1933 -                                                       file_name, line);
1934 +                       rw_lock_remove_debug_info(lock, pass,
1935 +                                              RW_LOCK_WAIT_EX);
1936  #endif
1937 -
1938 -                       return(RW_LOCK_WAIT_EX);
1939 +                        /* It is possible to wake when lock_word < 0.
1940 +                        We must pass the while-loop check to proceed.*/
1941 +               } else {
1942 +                       sync_array_free_cell(sync_primary_wait_array,
1943 +                                            index);
1944                 }
1945 +       }
1946 +       rw_x_spin_round_count += i;
1947 +}
1948
1949 -       } else if ((rw_lock_get_writer(lock) == RW_LOCK_WAIT_EX)
1950 -                  && os_thread_eq(lock->writer_thread,
1951 -                                               os_thread_get_curr_id())) {
1952 +/**********************************************************************
1953 +Low-level function for acquiring an exclusive lock. */
1954 +UNIV_INLINE
1955 +ibool
1956 +rw_lock_x_lock_low(
1957 +/*===============*/
1958 +                               /* out: RW_LOCK_NOT_LOCKED if did
1959 +                               not succeed, RW_LOCK_EX if success. */
1960 +       rw_lock_t*      lock,   /* in: pointer to rw-lock */
1961 +       ulint           pass,   /* in: pass value; != 0, if the lock will
1962 +                               be passed to another thread to unlock */
1963 +       const char*     file_name,/* in: file name where lock requested */
1964 +       ulint           line)   /* in: line where requested */
1965 +{
1966 +       os_thread_id_t  curr_thread     = os_thread_get_curr_id();
1967
1968 -               if (rw_lock_get_reader_count(lock) == 0) {
1969 +       if (rw_lock_lock_word_decr(lock, X_LOCK_DECR)) {
1970
1971 -                       rw_lock_set_writer(lock, RW_LOCK_EX);
1972 -                       lock->writer_count++;
1973 -                       lock->pass = pass;
1974 -                       lock->writer_is_wait_ex = FALSE;
1975 +               /* lock->recursive also tells us if the writer_thread
1976 +               field is stale or active. As we are going to write
1977 +               our own thread id in that field it must be that the
1978 +               current writer_thread value is not active. */
1979 +               ut_a(!lock->recursive);
1980
1981 +               /* Decrement occurred: we are writer or next-writer. */
1982 +               rw_lock_set_writer_id_and_recursion_flag(lock,
1983 +                                               pass ? FALSE : TRUE);
1984 +
1985 +               rw_lock_x_lock_wait(lock,
1986  #ifdef UNIV_SYNC_DEBUG
1987 -                       rw_lock_remove_debug_info(lock, pass, RW_LOCK_WAIT_EX);
1988 -                       rw_lock_add_debug_info(lock, pass, RW_LOCK_EX,
1989 -                                                       file_name, line);
1990 +                                   pass,
1991  #endif
1992 +                                    file_name, line);
1993
1994 -                       lock->last_x_file_name = file_name;
1995 -                       lock->last_x_line = line;
1996 -
1997 -                       /* Locking succeeded, we may return */
1998 -                       return(RW_LOCK_EX);
1999 +       } else {
2000 +               /* Decrement failed: relock or failed lock */
2001 +               if (!pass && lock->recursive
2002 +                   && os_thread_eq(lock->writer_thread, curr_thread)) {
2003 +                       /* Relock */
2004 +                        lock->lock_word -= X_LOCK_DECR;
2005 +               } else {
2006 +                       /* Another thread locked before us */
2007 +                       return(FALSE);
2008                 }
2009 -
2010 -               return(RW_LOCK_WAIT_EX);
2011 -
2012 -       } else if ((rw_lock_get_writer(lock) == RW_LOCK_EX)
2013 -                  && os_thread_eq(lock->writer_thread,
2014 -                                               os_thread_get_curr_id())
2015 -                  && (lock->pass == 0)
2016 -                  && (pass == 0)) {
2017 -
2018 -               lock->writer_count++;
2019 -
2020 +       }
2021  #ifdef UNIV_SYNC_DEBUG
2022 -               rw_lock_add_debug_info(lock, pass, RW_LOCK_EX, file_name,
2023 -                                                                       line);
2024 +       rw_lock_add_debug_info(lock, pass, RW_LOCK_EX,
2025 +                              file_name, line);
2026  #endif
2027 +       lock->last_x_file_name = file_name;
2028 +       lock->last_x_line = (unsigned int) line;
2029
2030 -               lock->last_x_file_name = file_name;
2031 -               lock->last_x_line = line;
2032 -
2033 -               /* Locking succeeded, we may return */
2034 -               return(RW_LOCK_EX);
2035 -       }
2036 -
2037 -       /* Locking did not succeed */
2038 -       return(RW_LOCK_NOT_LOCKED);
2039 +       return(TRUE);
2040  }
2041
2042  /**********************************************************************
2043 @@ -448,47 +578,30 @@
2044         ulint           line)   /* in: line where requested */
2045  {
2046          ulint  index;  /* index of the reserved wait cell */
2047 -        ulint  state;  /* lock state acquired */
2048          ulint  i;      /* spin round count */
2049 +       ibool   spinning = FALSE;
2050
2051          ut_ad(rw_lock_validate(lock));
2052
2053 -lock_loop:
2054 -        /* Acquire the mutex protecting the rw-lock fields */
2055 -       mutex_enter_fast(&(lock->mutex));
2056 -
2057 -       state = rw_lock_x_lock_low(lock, pass, file_name, line);
2058 +       i = 0;
2059
2060 -       mutex_exit(&(lock->mutex));
2061 +lock_loop:
2062
2063 -       if (state == RW_LOCK_EX) {
2064 +       if (rw_lock_x_lock_low(lock, pass, file_name, line)) {
2065 +               rw_x_spin_round_count += i;
2066
2067                 return; /* Locking succeeded */
2068
2069 -       } else if (state == RW_LOCK_NOT_LOCKED) {
2070 -
2071 -               /* Spin waiting for the writer field to become free */
2072 -               i = 0;
2073 -
2074 -               while (rw_lock_get_writer(lock) != RW_LOCK_NOT_LOCKED
2075 -                                                       && i < SYNC_SPIN_ROUNDS) {
2076 -                       if (srv_spin_wait_delay) {
2077 -                               ut_delay(ut_rnd_interval(0,
2078 -                                                       srv_spin_wait_delay));
2079 -                       }
2080 +       } else {
2081
2082 -                       i++;
2083 -               }
2084 -               if (i == SYNC_SPIN_ROUNDS) {
2085 -                       os_thread_yield();
2086 +                if (!spinning) {
2087 +                        spinning = TRUE;
2088 +                        rw_x_spin_wait_count++;
2089                 }
2090 -        } else if (state == RW_LOCK_WAIT_EX) {
2091
2092 -               /* Spin waiting for the reader count field to become zero */
2093 -               i = 0;
2094 -
2095 -               while (rw_lock_get_reader_count(lock) != 0
2096 -                                                       && i < SYNC_SPIN_ROUNDS) {
2097 +               /* Spin waiting for the lock_word to become free */
2098 +               while (i < SYNC_SPIN_ROUNDS
2099 +                      && lock->lock_word <= 0) {
2100                         if (srv_spin_wait_delay) {
2101                                 ut_delay(ut_rnd_interval(0,
2102                                                         srv_spin_wait_delay));
2103 @@ -498,12 +611,13 @@
2104                 }
2105                 if (i == SYNC_SPIN_ROUNDS) {
2106                         os_thread_yield();
2107 +               } else {
2108 +                       goto lock_loop;
2109                 }
2110 -        } else {
2111 -               i = 0; /* Eliminate a compiler warning */
2112 -               ut_error;
2113         }
2114
2115 +       rw_x_spin_round_count += i;
2116 +
2117         if (srv_print_latch_waits) {
2118                 fprintf(stderr,
2119         "Thread %lu spin wait rw-x-lock at %p cfile %s cline %lu rnds %lu\n",
2120 @@ -511,39 +625,20 @@
2121                 lock->cfile_name, (ulong) lock->cline, (ulong) i);
2122         }
2123
2124 -       rw_x_spin_wait_count++;
2125 -
2126 -        /* We try once again to obtain the lock. Acquire the mutex protecting
2127 -       the rw-lock fields */
2128 -
2129 -       mutex_enter(rw_lock_get_mutex(lock));
2130 -
2131 -       state = rw_lock_x_lock_low(lock, pass, file_name, line);
2132 -
2133 -       if (state == RW_LOCK_EX) {
2134 -               mutex_exit(rw_lock_get_mutex(lock));
2135 -
2136 -               return; /* Locking succeeded */
2137 -       }
2138 -
2139 -       rw_x_system_call_count++;
2140 -
2141          sync_array_reserve_cell(sync_primary_wait_array,
2142                                 lock,
2143 -#ifdef __WIN__
2144 -                               /* On windows RW_LOCK_WAIT_EX signifies
2145 -                               that this thread should wait on the
2146 -                               special wait_ex_event. */
2147 -                               (state == RW_LOCK_WAIT_EX)
2148 -                                ? RW_LOCK_WAIT_EX :
2149 -#endif
2150                                 RW_LOCK_EX,
2151                                 file_name, line,
2152                                 &index);
2153
2154 -       rw_lock_set_waiters(lock, 1);
2155 -
2156 -       mutex_exit(rw_lock_get_mutex(lock));
2157 +       /* Waiters must be set before checking lock_word, to ensure signal
2158 +       is sent. This could lead to a few unnecessary wake-up signals. */
2159 +       rw_lock_set_waiter_flag(lock);
2160 +
2161 +       if (rw_lock_x_lock_low(lock, pass, file_name, line)) {
2162 +               sync_array_free_cell(sync_primary_wait_array, index);
2163 +               return; /* Locking succeeded */
2164 +       }
2165
2166         if (srv_print_latch_waits) {
2167                 fprintf(stderr,
2168 @@ -552,11 +647,13 @@
2169                 lock->cfile_name, (ulong) lock->cline);
2170         }
2171
2172 -       rw_x_system_call_count++;
2173 +       /* these stats may not be accurate */
2174 +       lock->count_os_wait++;
2175         rw_x_os_wait_count++;
2176
2177          sync_array_wait_event(sync_primary_wait_array, index);
2178
2179 +       i = 0;
2180          goto lock_loop;
2181  }
2182
2183 @@ -697,7 +794,9 @@
2184         rw_lock_t*      lock,   /* in: rw-lock */
2185         ulint           level)  /* in: level */
2186  {
2187 +#ifdef UNIV_SYNC_DEBUG
2188         lock->level = level;
2189 +#endif /* UNIV_SYNC_DEBUG */
2190  }
2191
2192  #ifdef UNIV_SYNC_DEBUG
2193 @@ -718,7 +817,7 @@
2194         ut_ad(lock);
2195         ut_ad(rw_lock_validate(lock));
2196
2197 -       mutex_enter(&(lock->mutex));
2198 +       rw_lock_debug_mutex_enter();
2199
2200         info = UT_LIST_GET_FIRST(lock->debug_list);
2201
2202 @@ -728,7 +827,7 @@
2203                     && (info->pass == 0)
2204                     && (info->lock_type == lock_type)) {
2205
2206 -                       mutex_exit(&(lock->mutex));
2207 +                       rw_lock_debug_mutex_exit();
2208                         /* Found! */
2209
2210                         return(TRUE);
2211 @@ -736,7 +835,7 @@
2212
2213                 info = UT_LIST_GET_NEXT(list, info);
2214         }
2215 -       mutex_exit(&(lock->mutex));
2216 +       rw_lock_debug_mutex_exit();
2217
2218         return(FALSE);
2219  }
2220 @@ -758,22 +857,18 @@
2221         ut_ad(lock);
2222         ut_ad(rw_lock_validate(lock));
2223
2224 -       mutex_enter(&(lock->mutex));
2225 -
2226         if (lock_type == RW_LOCK_SHARED) {
2227 -               if (lock->reader_count > 0) {
2228 +               if (rw_lock_get_reader_count(lock) > 0) {
2229                         ret = TRUE;
2230                 }
2231         } else if (lock_type == RW_LOCK_EX) {
2232 -               if (lock->writer == RW_LOCK_EX) {
2233 +               if (rw_lock_get_writer(lock) == RW_LOCK_EX) {
2234                         ret = TRUE;
2235                 }
2236         } else {
2237                 ut_error;
2238         }
2239
2240 -       mutex_exit(&(lock->mutex));
2241 -
2242         return(ret);
2243  }
2244
2245 @@ -801,11 +896,10 @@
2246
2247                 count++;
2248
2249 +#ifndef HAVE_ATOMIC_BUILTINS
2250                 mutex_enter(&(lock->mutex));
2251 -
2252 -               if ((rw_lock_get_writer(lock) != RW_LOCK_NOT_LOCKED)
2253 -                   || (rw_lock_get_reader_count(lock) != 0)
2254 -                   || (rw_lock_get_waiters(lock) != 0)) {
2255 +#endif
2256 +               if (lock->lock_word != X_LOCK_DECR) {
2257
2258                         fprintf(stderr, "RW-LOCK: %p ", lock);
2259
2260 @@ -821,8 +915,10 @@
2261                                 info = UT_LIST_GET_NEXT(list, info);
2262                         }
2263                 }
2264 -
2265 +#ifndef HAVE_ATOMIC_BUILTINS
2266                 mutex_exit(&(lock->mutex));
2267 +#endif
2268 +
2269                 lock = UT_LIST_GET_NEXT(list, lock);
2270         }
2271
2272 @@ -845,9 +941,10 @@
2273                 "RW-LATCH INFO\n"
2274                 "RW-LATCH: %p ", lock);
2275
2276 -       if ((rw_lock_get_writer(lock) != RW_LOCK_NOT_LOCKED)
2277 -           || (rw_lock_get_reader_count(lock) != 0)
2278 -           || (rw_lock_get_waiters(lock) != 0)) {
2279 +#ifndef HAVE_ATOMIC_BUILTINS
2280 +       mutex_enter(&(lock->mutex));
2281 +#endif
2282 +       if (lock->lock_word != X_LOCK_DECR) {
2283
2284                 if (rw_lock_get_waiters(lock)) {
2285                         fputs(" Waiters for the lock exist\n", stderr);
2286 @@ -861,6 +958,9 @@
2287                         info = UT_LIST_GET_NEXT(list, info);
2288                 }
2289         }
2290 +#ifndef HAVE_ATOMIC_BUILTINS
2291 +       mutex_exit(&(lock->mutex));
2292 +#endif
2293  }
2294
2295  /*************************************************************************
2296 @@ -909,14 +1009,11 @@
2297         lock = UT_LIST_GET_FIRST(rw_lock_list);
2298
2299         while (lock != NULL) {
2300 -               mutex_enter(rw_lock_get_mutex(lock));
2301
2302 -               if ((rw_lock_get_writer(lock) != RW_LOCK_NOT_LOCKED)
2303 -                               || (rw_lock_get_reader_count(lock) != 0)) {
2304 +               if (lock->lock_word != X_LOCK_DECR) {
2305                         count++;
2306                 }
2307
2308 -               mutex_exit(rw_lock_get_mutex(lock));
2309                 lock = UT_LIST_GET_NEXT(list, lock);
2310         }
2311
2312 diff -ruN a/innobase/sync/sync0sync.c b/innobase/sync/sync0sync.c
2313 --- a/innobase/sync/sync0sync.c 2009-10-22 15:15:05.000000000 +0900
2314 +++ b/innobase/sync/sync0sync.c 2009-10-22 15:18:44.000000000 +0900
2315 @@ -1,8 +1,31 @@
2316 +/*****************************************************************************
2317 +
2318 +Copyright (c) 1995, 2009, Innobase Oy. All Rights Reserved.
2319 +Copyright (c) 2008, Google Inc.
2320 +
2321 +Portions of this file contain modifications contributed and copyrighted by
2322 +Google, Inc. Those modifications are gratefully acknowledged and are described
2323 +briefly in the InnoDB documentation. The contributions by Google are
2324 +incorporated with their permission, and subject to the conditions contained in
2325 +the file COPYING.Google.
2326 +
2327 +This program is free software; you can redistribute it and/or modify it under
2328 +the terms of the GNU General Public License as published by the Free Software
2329 +Foundation; version 2 of the License.
2330 +
2331 +This program is distributed in the hope that it will be useful, but WITHOUT
2332 +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
2333 +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
2334 +
2335 +You should have received a copy of the GNU General Public License along with
2336 +this program; if not, write to the Free Software Foundation, Inc., 59 Temple
2337 +Place, Suite 330, Boston, MA 02111-1307 USA
2338 +
2339 +*****************************************************************************/
2340 +
2341  /******************************************************
2342  Mutex, the basic synchronization primitive
2343
2344 -(c) 1995 Innobase Oy
2345 -
2346  Created 9/5/1995 Heikki Tuuri
2347  *******************************************************/
2348
2349 @@ -140,17 +163,12 @@
2350
2351  ulint  sync_dummy                      = 0;
2352
2353 -/* The number of system calls made in this module. Intended for performance
2354 -monitoring. */
2355 -
2356 -ulint  mutex_system_call_count         = 0;
2357 -
2358  /* Number of spin waits on mutexes: for performance monitoring */
2359
2360 -ulint  mutex_spin_round_count          = 0;
2361 -ulint  mutex_spin_wait_count           = 0;
2362 -ulint  mutex_os_wait_count             = 0;
2363 -ulint  mutex_exit_count                = 0;
2364 +ib_longlong    mutex_spin_round_count          = 0;
2365 +ib_longlong    mutex_spin_wait_count           = 0;
2366 +ib_longlong    mutex_os_wait_count             = 0;
2367 +ib_longlong    mutex_exit_count                = 0;
2368
2369  /* The global array of wait cells for implementation of the database's own
2370  mutexes and read-write locks */
2371 @@ -240,6 +258,8 @@
2372  {
2373  #if defined(_WIN32) && defined(UNIV_CAN_USE_X86_ASSEMBLER)
2374         mutex_reset_lock_word(mutex);
2375 +#elif defined(HAVE_ATOMIC_BUILTINS)
2376 +       mutex_reset_lock_word(mutex);
2377  #else
2378         os_fast_mutex_init(&(mutex->os_fast_mutex));
2379         mutex->lock_word = 0;
2380 @@ -325,7 +345,9 @@
2381
2382         os_event_free(mutex->event);
2383
2384 -#if !defined(_WIN32) || !defined(UNIV_CAN_USE_X86_ASSEMBLER)
2385 +#if defined(_WIN32) && defined(UNIV_CAN_USE_X86_ASSEMBLER)
2386 +#elif defined(HAVE_ATOMIC_BUILTINS)
2387 +#else
2388         os_fast_mutex_free(&(mutex->os_fast_mutex));
2389  #endif
2390         /* If we free the mutex protecting the mutex list (freeing is
2391 @@ -421,6 +443,12 @@
2392  #endif /* UNIV_DEBUG && !UNIV_HOTBACKUP */
2393    ut_ad(mutex);
2394
2395 +       /* This update is not thread safe, but we don't mind if the count
2396 +       isn't exact. Moved out of ifdef that follows because we are willing
2397 +       to sacrifice the cost of counting this as the data is valuable.
2398 +       Count the number of calls to mutex_spin_wait. */
2399 +       mutex_spin_wait_count++;
2400 +
2401  mutex_loop:
2402
2403    i = 0;
2404 @@ -433,7 +461,6 @@
2405
2406  spin_loop:
2407  #if defined UNIV_DEBUG && !defined UNIV_HOTBACKUP
2408 -  mutex_spin_wait_count++;
2409    mutex->count_spin_loop++;
2410  #endif /* UNIV_DEBUG && !UNIV_HOTBACKUP */
2411
2412 @@ -502,8 +529,6 @@
2413    sync_array_reserve_cell(sync_primary_wait_array, mutex,
2414                            SYNC_MUTEX, file_name, line, &index);
2415
2416 -  mutex_system_call_count++;
2417 -
2418    /* The memory order of the array reservation and the change in the
2419    waiters field is important: when we suspend a thread, we first
2420    reserve the cell and then set waiters field to 1. When threads are
2421 @@ -551,7 +576,6 @@
2422              mutex->cfile_name, (ulong) mutex->cline, (ulong) i);
2423  #endif
2424
2425 -  mutex_system_call_count++;
2426    mutex_os_wait_count++;
2427
2428  #ifndef UNIV_HOTBACKUP
2429 @@ -1368,20 +1392,31 @@
2430         FILE*   file)           /* in: file where to print */
2431  {
2432  #ifdef UNIV_SYNC_DEBUG
2433 -       fprintf(stderr, "Mutex exits %lu, rws exits %lu, rwx exits %lu\n",
2434 +       fprintf(file, "Mutex exits %llu, rws exits %llu, rwx exits %llu\n",
2435                 mutex_exit_count, rw_s_exit_count, rw_x_exit_count);
2436  #endif
2437
2438         fprintf(file,
2439 -"Mutex spin waits %lu, rounds %lu, OS waits %lu\n"
2440 -"RW-shared spins %lu, OS waits %lu; RW-excl spins %lu, OS waits %lu\n",
2441 -                       (ulong) mutex_spin_wait_count,
2442 -                       (ulong) mutex_spin_round_count,
2443 -                       (ulong) mutex_os_wait_count,
2444 -                       (ulong) rw_s_spin_wait_count,
2445 -                       (ulong) rw_s_os_wait_count,
2446 -                       (ulong) rw_x_spin_wait_count,
2447 -                       (ulong) rw_x_os_wait_count);
2448 +               "Mutex spin waits %llu, rounds %llu, OS waits %llu\n"
2449 +               "RW-shared spins %llu, OS waits %llu;"
2450 +               " RW-excl spins %llu, OS waits %llu\n",
2451 +               mutex_spin_wait_count,
2452 +               mutex_spin_round_count,
2453 +               mutex_os_wait_count,
2454 +               rw_s_spin_wait_count,
2455 +               rw_s_os_wait_count,
2456 +               rw_x_spin_wait_count,
2457 +               rw_x_os_wait_count);
2458 +
2459 +       fprintf(file,
2460 +               "Spin rounds per wait: %.2f mutex, %.2f RW-shared, "
2461 +               "%.2f RW-excl\n",
2462 +               (double) mutex_spin_round_count /
2463 +               (mutex_spin_wait_count ? mutex_spin_wait_count : 1),
2464 +               (double) rw_s_spin_round_count /
2465 +               (rw_s_spin_wait_count ? rw_s_spin_wait_count : 1),
2466 +               (double) rw_x_spin_round_count /
2467 +               (rw_x_spin_wait_count ? rw_x_spin_wait_count : 1));
2468  }
2469
2470  /***********************************************************************
2471 diff -ruN a/patch_info/innodb_rw_lock.info b/patch_info/innodb_rw_lock.info
2472 --- /dev/null   1970-01-01 09:00:00.000000000 +0900
2473 +++ b/patch_info/innodb_rw_lock.info    2009-10-22 15:18:30.000000000 +0900
2474 @@ -0,0 +1,6 @@
2475 +File=innodb_rw_lock.patch
2476 +Name=Fix of InnoDB rw_locks ported from InnoDB Plugin
2477 +Version=1.0
2478 +Author=InnoBase Oy.
2479 +License=GPL
2480 +Comment=