mysql-innodb_rw_lock.patch

   1 diff -ruN a/innobase/btr/btr0sea.c b/innobase/btr/btr0sea.c
   2 --- a/innobase/btr/btr0sea.c    2009-05-20 14:21:44.000000000 +0900
   3 +++ b/innobase/btr/btr0sea.c    2009-05-20 14:39:34.000000000 +0900
   4 @@ -773,7 +773,7 @@
   5                 rw_lock_s_lock(&btr_search_latch);
   6         }
   7
   8 -       ut_ad(btr_search_latch.writer != RW_LOCK_EX);
   9 +       ut_ad(btr_search_latch.writer_count == 0);
  10         ut_ad(btr_search_latch.reader_count > 0);
  11
  12         rec = ha_search_and_get_data(btr_search_sys->hash_index, fold);
  13 diff -ruN a/innobase/include/sync0rw.h b/innobase/include/sync0rw.h
  14 --- a/innobase/include/sync0rw.h        2009-01-30 06:42:20.000000000 +0900
  15 +++ b/innobase/include/sync0rw.h        2009-04-16 16:15:28.000000000 +0900
  16 @@ -325,7 +325,17 @@
  17  Accessor functions for rw lock. */
  18  UNIV_INLINE
  19  ulint
  20 -rw_lock_get_waiters(
  21 +rw_lock_get_s_waiters(
  22 +/*==================*/
  23 +       rw_lock_t*      lock);
  24 +UNIV_INLINE
  25 +ulint
  26 +rw_lock_get_x_waiters(
  27 +/*==================*/
  28 +       rw_lock_t*      lock);
  29 +UNIV_INLINE
  30 +ulint
  31 +rw_lock_get_wx_waiters(
  32  /*================*/
  33         rw_lock_t*      lock);
  34  UNIV_INLINE
  35 @@ -408,6 +418,17 @@
  36         rw_lock_debug_t*        info);  /* in: debug struct */
  37  #endif /* UNIV_SYNC_DEBUG */
  38
  39 +#ifdef HAVE_ATOMIC_BUILTINS
  40 +/* This value means NOT_LOCKED */
  41 +#define RW_LOCK_BIAS           0x00100000
  42 +#else
  43 +#error HAVE_ATOMIC_BUILTINS is not defined. Do you use enough new GCC or compatibles?
  44 +#error Or do you use exact options for CFLAGS?
  45 +#error e.g. (for x86_32): "-m32 -march=i586 -mtune=i686"
  46 +#error e.g. (for Sparc_64): "-m64 -mcpu=v9"
  47 +#error Otherwise, this build may be slower than normal version.
  48 +#endif
  49 +
  50  /* NOTE! The structure appears here only for the compiler to know its size.
  51  Do not use its fields directly! The structure used in the spin lock
  52  implementation of a read-write lock. Several threads may have a shared lock
  53 @@ -417,9 +438,9 @@
  54  field. Then no new readers are allowed in. */
  55
  56  struct rw_lock_struct {
  57 -       os_event_t      event;  /* Used by sync0arr.c for thread queueing */
  58 -
  59 -#ifdef __WIN__
  60 +                       /* Used by sync0arr.c for thread queueing */
  61 +       os_event_t      s_event;        /* Used for s_lock */
  62 +       os_event_t      x_event;        /* Used for x_lock */
  63         os_event_t      wait_ex_event;  /* This windows specific event is
  64                                 used by the thread which has set the
  65                                 lock state to RW_LOCK_WAIT_EX. The
  66 @@ -427,31 +448,35 @@
  67                                 thread will be the next one to proceed
  68                                 once the current the event gets
  69                                 signalled. See LEMMA 2 in sync0sync.c */
  70 +
  71 +#ifdef HAVE_ATOMIC_BUILTINS
  72 +       volatile lint   lock_word;      /* Used by using atomic builtin */
  73  #endif
  74
  75 -       ulint   reader_count;   /* Number of readers who have locked this
  76 +       volatile ulint  reader_count;   /* Number of readers who have locked this
  77                                 lock in the shared mode */
  78 -       ulint   writer;         /* This field is set to RW_LOCK_EX if there
  79 +       volatile ulint  writer;         /* This field is set to RW_LOCK_EX if there
  80                                 is a writer owning the lock (in exclusive
  81                                 mode), RW_LOCK_WAIT_EX if a writer is
  82                                 queueing for the lock, and
  83                                 RW_LOCK_NOT_LOCKED, otherwise. */
  84 -       os_thread_id_t  writer_thread;
  85 +       volatile os_thread_id_t writer_thread;
  86                                 /* Thread id of a possible writer thread */
  87 -       ulint   writer_count;   /* Number of times the same thread has
  88 +       volatile ulint  writer_count;   /* Number of times the same thread has
  89                                 recursively locked the lock in the exclusive
  90                                 mode */
  91 +#ifndef HAVE_ATOMIC_BUILTINS
  92         mutex_t mutex;          /* The mutex protecting rw_lock_struct */
  93 +#endif
  94         ulint   pass;           /* Default value 0. This is set to some
  95                                 value != 0 given by the caller of an x-lock
  96                                 operation, if the x-lock is to be passed to
  97                                 another thread to unlock (which happens in
  98                                 asynchronous i/o). */
  99 -       ulint   waiters;        /* This ulint is set to 1 if there are
 100 -                               waiters (readers or writers) in the global
 101 -                               wait array, waiting for this rw_lock.
 102 -                               Otherwise, == 0. */
 103 -       ibool   writer_is_wait_ex;
 104 +       volatile ulint  s_waiters; /* 1: there are waiters (s_lock) */
 105 +       volatile ulint  x_waiters; /* 1: there are waiters (x_lock) */
 106 +       volatile ulint  wait_ex_waiters; /* 1: there are waiters (wait_ex) */
 107 +       volatile ibool  writer_is_wait_ex;
 108                                 /* This is TRUE if the writer field is
 109                                 RW_LOCK_WAIT_EX; this field is located far
 110                                 from the memory update hotspot fields which
 111 diff -ruN a/innobase/include/sync0rw.ic b/innobase/include/sync0rw.ic
 112 --- a/innobase/include/sync0rw.ic       2009-01-30 06:42:20.000000000 +0900
 113 +++ b/innobase/include/sync0rw.ic       2009-04-16 17:06:53.000000000 +0900
 114 @@ -47,20 +47,64 @@
 115  Accessor functions for rw lock. */
 116  UNIV_INLINE
 117  ulint
 118 -rw_lock_get_waiters(
 119 +rw_lock_get_s_waiters(
 120  /*================*/
 121         rw_lock_t*      lock)
 122  {
 123 -       return(lock->waiters);
 124 +       return(lock->s_waiters);
 125  }
 126  UNIV_INLINE
 127 -void
 128 -rw_lock_set_waiters(
 129 +ulint
 130 +rw_lock_get_x_waiters(
 131  /*================*/
 132 +       rw_lock_t*      lock)
 133 +{
 134 +       return(lock->x_waiters);
 135 +}
 136 +UNIV_INLINE
 137 +ulint
 138 +rw_lock_get_wx_waiters(
 139 +/*================*/
 140 +       rw_lock_t*      lock)
 141 +{
 142 +       return(lock->wait_ex_waiters);
 143 +}
 144 +UNIV_INLINE
 145 +void
 146 +rw_lock_set_s_waiters(
 147         rw_lock_t*      lock,
 148         ulint           flag)
 149  {
 150 -       lock->waiters = flag;
 151 +#ifdef HAVE_ATOMIC_BUILTINS
 152 +       __sync_lock_test_and_set(&lock->s_waiters, flag);
 153 +#else
 154 +       lock->s_waiters = flag;
 155 +#endif
 156 +}
 157 +UNIV_INLINE
 158 +void
 159 +rw_lock_set_x_waiters(
 160 +       rw_lock_t*      lock,
 161 +       ulint           flag)
 162 +{
 163 +#ifdef HAVE_ATOMIC_BUILTINS
 164 +       __sync_lock_test_and_set(&lock->x_waiters, flag);
 165 +#else
 166 +       lock->x_waiters = flag;
 167 +#endif
 168 +}
 169 +UNIV_INLINE
 170 +void
 171 +rw_lock_set_wx_waiters(
 172 +/*================*/
 173 +       rw_lock_t*      lock,
 174 +       ulint           flag)
 175 +{
 176 +#ifdef HAVE_ATOMIC_BUILTINS
 177 +       __sync_lock_test_and_set(&lock->wait_ex_waiters, flag);
 178 +#else
 179 +       lock->wait_ex_waiters = flag;
 180 +#endif
 181  }
 182  UNIV_INLINE
 183  ulint
 184 @@ -68,7 +112,19 @@
 185  /*===============*/
 186         rw_lock_t*      lock)
 187  {
 188 +#ifdef HAVE_ATOMIC_BUILTINS
 189 +       if (lock->writer == RW_LOCK_NOT_LOCKED) {
 190 +               return(RW_LOCK_NOT_LOCKED);
 191 +       }
 192 +
 193 +       if (lock->writer_is_wait_ex) {
 194 +               return(RW_LOCK_WAIT_EX);
 195 +       } else {
 196 +               return(RW_LOCK_EX);
 197 +       }
 198 +#else
 199         return(lock->writer);
 200 +#endif
 201  }
 202  UNIV_INLINE
 203  void
 204 @@ -96,6 +152,7 @@
 205  {
 206         lock->reader_count = count;
 207  }
 208 +#ifndef HAVE_ATOMIC_BUILTINS
 209  UNIV_INLINE
 210  mutex_t*
 211  rw_lock_get_mutex(
 212 @@ -104,6 +161,7 @@
 213  {
 214         return(&(lock->mutex));
 215  }
 216 +#endif
 217
 218  /**********************************************************************
 219  Returns the value of writer_count for the lock. Does not reserve the lock
 220 @@ -133,14 +191,26 @@
 221         const char*     file_name, /* in: file name where lock requested */
 222         ulint           line)   /* in: line where requested */
 223  {
 224 -#ifdef UNIV_SYNC_DEBUG
 225 +#if defined(UNIV_SYNC_DEBUG) && !defined(HAVE_ATOMIC_BUILTINS)
 226         ut_ad(mutex_own(rw_lock_get_mutex(lock)));
 227  #endif /* UNIV_SYNC_DEBUG */
 228         /* Check if the writer field is free */
 229
 230 +#ifdef HAVE_ATOMIC_BUILTINS
 231 +       if (UNIV_LIKELY(rw_lock_get_writer(lock) == RW_LOCK_NOT_LOCKED)) {
 232 +               /* try s-lock */
 233 +               if(__sync_sub_and_fetch(&(lock->lock_word),1) <= 0) {
 234 +                       /* fail */
 235 +                       __sync_fetch_and_add(&(lock->lock_word),1);
 236 +                       return(FALSE);  /* locking did not succeed */
 237 +               }
 238 +               /* success */
 239 +               __sync_fetch_and_add(&(lock->reader_count),1);
 240 +#else
 241         if (UNIV_LIKELY(lock->writer == RW_LOCK_NOT_LOCKED)) {
 242                 /* Set the shared lock by incrementing the reader count */
 243                 lock->reader_count++;
 244 +#endif
 245
 246  #ifdef UNIV_SYNC_DEBUG
 247                 rw_lock_add_debug_info(lock, pass, RW_LOCK_SHARED, file_name,
 248 @@ -167,11 +237,15 @@
 249         const char*     file_name,      /* in: file name where requested */
 250         ulint           line)           /* in: line where lock requested */
 251  {
 252 -       ut_ad(lock->writer == RW_LOCK_NOT_LOCKED);
 253 +       ut_ad(rw_lock_get_writer(lock) == RW_LOCK_NOT_LOCKED);
 254         ut_ad(rw_lock_get_reader_count(lock) == 0);
 255
 256         /* Set the shared lock by incrementing the reader count */
 257 +#ifdef HAVE_ATOMIC_BUILTINS
 258 +       __sync_fetch_and_add(&(lock->reader_count),1);
 259 +#else
 260         lock->reader_count++;
 261 +#endif
 262
 263         lock->last_s_file_name = file_name;
 264         lock->last_s_line = line;
 265 @@ -199,7 +273,11 @@
 266
 267         rw_lock_set_writer(lock, RW_LOCK_EX);
 268         lock->writer_thread = os_thread_get_curr_id();
 269 +#ifdef HAVE_ATOMIC_BUILTINS
 270 +       __sync_fetch_and_add(&(lock->writer_count),1);
 271 +#else
 272         lock->writer_count++;
 273 +#endif
 274         lock->pass = 0;
 275
 276         lock->last_x_file_name = file_name;
 277 @@ -241,15 +319,21 @@
 278         ut_ad(!rw_lock_own(lock, RW_LOCK_SHARED)); /* see NOTE above */
 279  #endif /* UNIV_SYNC_DEBUG */
 280
 281 +#ifndef HAVE_ATOMIC_BUILTINS
 282         mutex_enter(rw_lock_get_mutex(lock));
 283 +#endif
 284
 285         if (UNIV_LIKELY(rw_lock_s_lock_low(lock, pass, file_name, line))) {
 286 +#ifndef HAVE_ATOMIC_BUILTINS
 287                 mutex_exit(rw_lock_get_mutex(lock));
 288 +#endif
 289
 290                 return; /* Success */
 291         } else {
 292                 /* Did not succeed, try spin wait */
 293 +#ifndef HAVE_ATOMIC_BUILTINS
 294                 mutex_exit(rw_lock_get_mutex(lock));
 295 +#endif
 296
 297                 rw_lock_s_lock_spin(lock, pass, file_name, line);
 298
 299 @@ -272,11 +356,23 @@
 300  {
 301         ibool   success = FALSE;
 302
 303 +#ifdef HAVE_ATOMIC_BUILTINS
 304 +       if (rw_lock_get_writer(lock) == RW_LOCK_NOT_LOCKED) {
 305 +               /* try s-lock */
 306 +               if(__sync_sub_and_fetch(&(lock->lock_word),1) <= 0) {
 307 +                       /* fail */
 308 +                       __sync_fetch_and_add(&(lock->lock_word),1);
 309 +                       return(FALSE);  /* locking did not succeed */
 310 +               }
 311 +               /* success */
 312 +               __sync_fetch_and_add(&(lock->reader_count),1);
 313 +#else
 314         mutex_enter(rw_lock_get_mutex(lock));
 315
 316         if (lock->writer == RW_LOCK_NOT_LOCKED) {
 317                 /* Set the shared lock by incrementing the reader count */
 318                 lock->reader_count++;
 319 +#endif
 320
 321  #ifdef UNIV_SYNC_DEBUG
 322                 rw_lock_add_debug_info(lock, 0, RW_LOCK_SHARED, file_name,
 323 @@ -289,7 +385,9 @@
 324                 success = TRUE;
 325         }
 326
 327 +#ifndef HAVE_ATOMIC_BUILTINS
 328         mutex_exit(rw_lock_get_mutex(lock));
 329 +#endif
 330
 331         return(success);
 332  }
 333 @@ -309,6 +407,54 @@
 334  {
 335         ibool           success         = FALSE;
 336         os_thread_id_t  curr_thread     = os_thread_get_curr_id();
 337 +#ifdef HAVE_ATOMIC_BUILTINS
 338 +       if (lock->reader_count == 0) {
 339 +               /* try to lock writer */
 340 +               if(__sync_lock_test_and_set(&(lock->writer),RW_LOCK_EX)
 341 +                               == RW_LOCK_NOT_LOCKED) {
 342 +                       /* success */
 343 +retry_x_lock:
 344 +                       /* try x-lock */
 345 +                       if(__sync_sub_and_fetch(&(lock->lock_word),
 346 +                                       RW_LOCK_BIAS) == 0) {
 347 +                               /* success */
 348 +                               lock->writer_thread = curr_thread;
 349 +                               lock->pass = 0;
 350 +                               lock->writer_is_wait_ex = FALSE;
 351 +                               /* next function may work as memory barrier */
 352 +                       relock:
 353 +                               __sync_fetch_and_add(&(lock->writer_count),1);
 354 +
 355 +#ifdef UNIV_SYNC_DEBUG
 356 +                               rw_lock_add_debug_info(lock, 0, RW_LOCK_EX, file_name, line);
 357 +#endif
 358 +
 359 +                               lock->last_x_file_name = file_name;
 360 +                               lock->last_x_line = line;
 361 +
 362 +                               ut_ad(rw_lock_validate(lock));
 363 +
 364 +                               return(TRUE);
 365 +                       } else {
 366 +                               /* fail (x-lock) */
 367 +                               if (__sync_fetch_and_add(&(lock->lock_word),RW_LOCK_BIAS)
 368 +                                               == 0)
 369 +                                       goto retry_x_lock;
 370 +                       }
 371 +
 372 +                       __sync_lock_test_and_set(&(lock->writer),RW_LOCK_NOT_LOCKED);
 373 +               }
 374 +       }
 375 +
 376 +       if (lock->pass == 0
 377 +                       && os_thread_eq(lock->writer_thread, curr_thread)) {
 378 +               goto relock;
 379 +       }
 380 +
 381 +       //ut_ad(rw_lock_validate(lock));
 382 +
 383 +       return(FALSE);
 384 +#else
 385         mutex_enter(rw_lock_get_mutex(lock));
 386
 387         if (UNIV_UNLIKELY(rw_lock_get_reader_count(lock) != 0)) {
 388 @@ -339,6 +485,7 @@
 389          ut_ad(rw_lock_validate(lock));
 390
 391         return(success);
 392 +#endif
 393  }
 394
 395  /**********************************************************************
 396 @@ -354,16 +501,33 @@
 397  #endif
 398         )
 399  {
 400 +#ifndef HAVE_ATOMIC_BUILTINS
 401         mutex_t*        mutex   = &(lock->mutex);
 402 -       ibool           sg      = FALSE;
 403 +#endif
 404 +       ibool           x_sg    = FALSE;
 405 +       ibool           wx_sg   = FALSE;
 406 +#ifdef HAVE_ATOMIC_BUILTINS
 407 +       ibool           last    = FALSE;
 408 +#endif
 409
 410 +#ifndef HAVE_ATOMIC_BUILTINS
 411          /* Acquire the mutex protecting the rw-lock fields */
 412         mutex_enter(mutex);
 413 +#endif
 414
 415         /* Reset the shared lock by decrementing the reader count */
 416
 417         ut_a(lock->reader_count > 0);
 418 +#ifdef HAVE_ATOMIC_BUILTINS
 419 +       /* unlock lock_word */
 420 +       __sync_fetch_and_add(&(lock->lock_word),1);
 421 +
 422 +       if(__sync_sub_and_fetch(&(lock->reader_count),1) == 0) {
 423 +               last = TRUE;
 424 +       }
 425 +#else
 426         lock->reader_count--;
 427 +#endif
 428
 429  #ifdef UNIV_SYNC_DEBUG
 430         rw_lock_remove_debug_info(lock, pass, RW_LOCK_SHARED);
 431 @@ -372,22 +536,39 @@
 432         /* If there may be waiters and this was the last s-lock,
 433         signal the object */
 434
 435 -       if (UNIV_UNLIKELY(lock->waiters)
 436 +#ifdef HAVE_ATOMIC_BUILTINS
 437 +       if (UNIV_UNLIKELY(last && __sync_lock_test_and_set(&lock->wait_ex_waiters, 0))) {
 438 +               os_event_set(lock->wait_ex_event);
 439 +               sync_array_object_signalled(sync_primary_wait_array);
 440 +       }
 441 +       else if (UNIV_UNLIKELY(last && __sync_lock_test_and_set(&lock->x_waiters, 0))) {
 442 +               os_event_set(lock->x_event);
 443 +               sync_array_object_signalled(sync_primary_wait_array);
 444 +       }
 445 +#else
 446 +       if (UNIV_UNLIKELY(lock->wait_ex_waiters)
 447                         && lock->reader_count == 0) {
 448 -               sg = TRUE;
 449 +               wx_sg = TRUE;
 450
 451 -               rw_lock_set_waiters(lock, 0);
 452 +               rw_lock_set_wx_waiters(lock, 0);
 453 +       }
 454 +       else if (UNIV_UNLIKELY(lock->x_waiters)
 455 +                       && lock->reader_count == 0) {
 456 +               x_sg = TRUE;
 457 +
 458 +               rw_lock_set_x_waiters(lock, 0);
 459         }
 460
 461         mutex_exit(mutex);
 462
 463 -       if (UNIV_UNLIKELY(sg)) {
 464 -#ifdef __WIN__
 465 +       if (UNIV_UNLIKELY(wx_sg)) {
 466                 os_event_set(lock->wait_ex_event);
 467 -#endif
 468 -               os_event_set(lock->event);
 469 +               sync_array_object_signalled(sync_primary_wait_array);
 470 +       } else if (UNIV_UNLIKELY(x_sg)) {
 471 +               os_event_set(lock->x_event);
 472                 sync_array_object_signalled(sync_primary_wait_array);
 473         }
 474 +#endif
 475
 476          ut_ad(rw_lock_validate(lock));
 477
 478 @@ -409,13 +590,22 @@
 479
 480         ut_ad(lock->reader_count > 0);
 481
 482 +#ifdef HAVE_ATOMIC_BUILTINS
 483 +       __sync_sub_and_fetch(&(lock->reader_count),1);
 484 +#else
 485         lock->reader_count--;
 486 +#endif
 487
 488  #ifdef UNIV_SYNC_DEBUG
 489         rw_lock_remove_debug_info(lock, 0, RW_LOCK_SHARED);
 490  #endif
 491
 492 +#ifdef HAVE_ATOMIC_BUILTINS
 493 +       ut_ad(!lock->s_waiters);
 494 +       ut_ad(!lock->x_waiters);
 495 +#else
 496         ut_ad(!lock->waiters);
 497 +#endif
 498          ut_ad(rw_lock_validate(lock));
 499  #ifdef UNIV_SYNC_PERF_STAT
 500         rw_s_exit_count++;
 501 @@ -435,41 +625,83 @@
 502  #endif
 503         )
 504  {
 505 -       ibool   sg      = FALSE;
 506 +#ifdef HAVE_ATOMIC_BUILTINS
 507 +       ibool   last    = FALSE;
 508 +#endif
 509 +       ibool   s_sg    = FALSE;
 510 +       ibool   x_sg    = FALSE;
 511
 512 +#ifndef HAVE_ATOMIC_BUILTINS
 513          /* Acquire the mutex protecting the rw-lock fields */
 514         mutex_enter(&(lock->mutex));
 515 +#endif
 516
 517         /* Reset the exclusive lock if this thread no longer has an x-mode
 518         lock */
 519
 520         ut_ad(lock->writer_count > 0);
 521
 522 +#ifdef HAVE_ATOMIC_BUILTINS
 523 +       if(__sync_sub_and_fetch(&(lock->writer_count),1) == 0) {
 524 +               last = TRUE;
 525 +       }
 526 +
 527 +       if (last) {
 528 +               /* unlock lock_word */
 529 +               __sync_fetch_and_add(&(lock->lock_word),RW_LOCK_BIAS);
 530 +
 531 +               /* FIXME: It is a value of bad manners for pthread.
 532 +                         But we shouldn't keep an ID of not-owner. */
 533 +               lock->writer_thread = -1;
 534 +               __sync_lock_test_and_set(&(lock->writer),RW_LOCK_NOT_LOCKED);
 535 +       }
 536 +#else
 537         lock->writer_count--;
 538
 539         if (lock->writer_count == 0) {
 540                 rw_lock_set_writer(lock, RW_LOCK_NOT_LOCKED);
 541         }
 542 +#endif
 543
 544  #ifdef UNIV_SYNC_DEBUG
 545         rw_lock_remove_debug_info(lock, pass, RW_LOCK_EX);
 546  #endif
 547
 548         /* If there may be waiters, signal the lock */
 549 -       if (UNIV_UNLIKELY(lock->waiters)
 550 -                       && lock->writer_count == 0) {
 551 -
 552 -               sg = TRUE;
 553 -               rw_lock_set_waiters(lock, 0);
 554 +#ifdef HAVE_ATOMIC_BUILTINS
 555 +       if (last) {
 556 +               if(__sync_lock_test_and_set(&lock->s_waiters, 0)){
 557 +                       s_sg = TRUE;
 558 +               }
 559 +               if(__sync_lock_test_and_set(&lock->x_waiters, 0)){
 560 +                       x_sg = TRUE;
 561 +               }
 562 +       }
 563 +#else
 564 +       if (lock->writer_count == 0) {
 565 +               if(lock->s_waiters){
 566 +                       s_sg = TRUE;
 567 +                       rw_lock_set_s_waiters(lock, 0);
 568 +               }
 569 +               if(lock->x_waiters){
 570 +                       x_sg = TRUE;
 571 +                       rw_lock_set_x_waiters(lock, 0);
 572 +               }
 573         }
 574
 575         mutex_exit(&(lock->mutex));
 576 +#endif
 577
 578 -       if (UNIV_UNLIKELY(sg)) {
 579 +       if (UNIV_UNLIKELY(s_sg)) {
 580 +               os_event_set(lock->s_event);
 581 +               sync_array_object_signalled(sync_primary_wait_array);
 582 +       }
 583 +       if (UNIV_UNLIKELY(x_sg)) {
 584  #ifdef __WIN__
 585 +               /* I doubt the necessity of it. */
 586                 os_event_set(lock->wait_ex_event);
 587  #endif
 588 -               os_event_set(lock->event);
 589 +               os_event_set(lock->x_event);
 590                 sync_array_object_signalled(sync_primary_wait_array);
 591         }
 592
 593 @@ -494,9 +726,13 @@
 594
 595         ut_ad(lock->writer_count > 0);
 596
 597 +#ifdef HAVE_ATOMIC_BUILTINS
 598 +       if(__sync_sub_and_fetch(&(lock->writer_count),1) == 0) {
 599 +#else
 600         lock->writer_count--;
 601
 602         if (lock->writer_count == 0) {
 603 +#endif
 604                 rw_lock_set_writer(lock, RW_LOCK_NOT_LOCKED);
 605         }
 606
 607 @@ -504,7 +740,12 @@
 608         rw_lock_remove_debug_info(lock, 0, RW_LOCK_EX);
 609  #endif
 610
 611 +#ifdef HAVE_ATOMIC_BUILTINS
 612 +       ut_ad(!lock->s_waiters);
 613 +       ut_ad(!lock->x_waiters);
 614 +#else
 615         ut_ad(!lock->waiters);
 616 +#endif
 617          ut_ad(rw_lock_validate(lock));
 618
 619  #ifdef UNIV_SYNC_PERF_STAT
 620 diff -ruN a/innobase/sync/sync0arr.c b/innobase/sync/sync0arr.c
 621 --- a/innobase/sync/sync0arr.c  2009-01-30 06:42:24.000000000 +0900
 622 +++ b/innobase/sync/sync0arr.c  2009-04-16 16:15:28.000000000 +0900
 623 @@ -309,13 +309,13 @@
 624  {
 625         if (type == SYNC_MUTEX) {
 626                 return(os_event_reset(((mutex_t *) object)->event));
 627 -#ifdef __WIN__
 628         } else if (type == RW_LOCK_WAIT_EX) {
 629                 return(os_event_reset(
 630                        ((rw_lock_t *) object)->wait_ex_event));
 631 -#endif
 632 -       } else {
 633 -               return(os_event_reset(((rw_lock_t *) object)->event));
 634 +       } else if (type == RW_LOCK_SHARED) {
 635 +               return(os_event_reset(((rw_lock_t *) object)->s_event));
 636 +       } else { /* RW_LOCK_EX */
 637 +               return(os_event_reset(((rw_lock_t *) object)->x_event));
 638         }
 639  }
 640
 641 @@ -415,15 +415,12 @@
 642
 643         if (cell->request_type == SYNC_MUTEX) {
 644                 event = ((mutex_t*) cell->wait_object)->event;
 645 -#ifdef __WIN__
 646 -       /* On windows if the thread about to wait is the one which
 647 -       has set the state of the rw_lock to RW_LOCK_WAIT_EX, then
 648 -       it waits on a special event i.e.: wait_ex_event. */
 649         } else if (cell->request_type == RW_LOCK_WAIT_EX) {
 650                 event = ((rw_lock_t*) cell->wait_object)->wait_ex_event;
 651 -#endif
 652 -       } else {
 653 -               event = ((rw_lock_t*) cell->wait_object)->event;
 654 +       } else if (cell->request_type == RW_LOCK_SHARED) {
 655 +               event = ((rw_lock_t*) cell->wait_object)->s_event;
 656 +       } else {
 657 +               event = ((rw_lock_t*) cell->wait_object)->x_event;
 658         }
 659
 660                 cell->waiting = TRUE;
 661 @@ -464,6 +461,7 @@
 662         mutex_t*        mutex;
 663         rw_lock_t*      rwlock;
 664         ulint           type;
 665 +       ulint           writer;
 666
 667         type = cell->request_type;
 668
 669 @@ -492,12 +490,10 @@
 670                         (ulong) mutex->waiters);
 671
 672         } else if (type == RW_LOCK_EX
 673 -#ifdef __WIN__
 674                    || type == RW_LOCK_WAIT_EX
 675 -#endif
 676                    || type == RW_LOCK_SHARED) {
 677
 678 -               fputs(type == RW_LOCK_EX ? "X-lock on" : "S-lock on", file);
 679 +               fputs(type == RW_LOCK_SHARED ? "S-lock on" : "X-lock on", file);
 680
 681                 rwlock = cell->old_wait_rw_lock;
 682
 683 @@ -505,21 +501,23 @@
 684                         " RW-latch at %p created in file %s line %lu\n",
 685                         rwlock, rwlock->cfile_name,
 686                         (ulong) rwlock->cline);
 687 -               if (rwlock->writer != RW_LOCK_NOT_LOCKED) {
 688 +               writer = rw_lock_get_writer(rwlock);
 689 +               if (writer != RW_LOCK_NOT_LOCKED) {
 690                         fprintf(file,
 691                         "a writer (thread id %lu) has reserved it in mode %s",
 692                                 (ulong) os_thread_pf(rwlock->writer_thread),
 693 -                               rwlock->writer == RW_LOCK_EX
 694 +                               writer == RW_LOCK_EX
 695                                 ? " exclusive\n"
 696                                 : " wait exclusive\n");
 697                 }
 698
 699                 fprintf(file,
 700 -                       "number of readers %lu, waiters flag %lu\n"
 701 +                       "number of readers %lu, s_waiters flag %lu, x_waiters flag %lu\n"
 702                         "Last time read locked in file %s line %lu\n"
 703                         "Last time write locked in file %s line %lu\n",
 704                         (ulong) rwlock->reader_count,
 705 -                       (ulong) rwlock->waiters,
 706 +                       (ulong) rwlock->s_waiters,
 707 +                       (ulong) (rwlock->x_waiters || rwlock->wait_ex_waiters),
 708                         rwlock->last_s_file_name,
 709                         (ulong) rwlock->last_s_line,
 710                         rwlock->last_x_file_name,
 711 @@ -839,11 +837,15 @@
 712  /*========================*/
 713         sync_array_t*   arr)    /* in: wait array */
 714  {
 715 +#ifdef HAVE_ATOMIC_BUILTINS
 716 +       __sync_fetch_and_add(&(arr->sg_count),1);
 717 +#else
 718          sync_array_enter(arr);
 719
 720         arr->sg_count++;
 721
 722          sync_array_exit(arr);
 723 +#endif
 724  }
 725
 726  /**************************************************************************
 727 @@ -880,19 +882,23 @@
 728
 729                                         mutex = cell->wait_object;
 730                                         os_event_set(mutex->event);
 731 -#ifdef __WIN__
 732                                 } else if (cell->request_type
 733                                            == RW_LOCK_WAIT_EX) {
 734                                         rw_lock_t*      lock;
 735
 736                                         lock = cell->wait_object;
 737                                         os_event_set(lock->wait_ex_event);
 738 -#endif
 739 -                               } else {
 740 +                               } else if (cell->request_type
 741 +                                          == RW_LOCK_SHARED) {
 742                                         rw_lock_t*      lock;
 743
 744                                         lock = cell->wait_object;
 745 -                                       os_event_set(lock->event);
 746 +                                       os_event_set(lock->s_event);
 747 +                               } else {
 748 +                                       rw_lock_t*      lock;
 749 +
 750 +                                       lock = cell->wait_object;
 751 +                                       os_event_set(lock->x_event);
 752                                 }
 753                          }
 754                  }
 755 diff -ruN a/innobase/sync/sync0rw.c b/innobase/sync/sync0rw.c
 756 --- a/innobase/sync/sync0rw.c   2009-01-30 06:42:24.000000000 +0900
 757 +++ b/innobase/sync/sync0rw.c   2009-04-16 17:33:59.000000000 +0900
 758 @@ -99,6 +99,7 @@
 759         object is created, then the following call initializes
 760         the sync system. */
 761
 762 +#ifndef HAVE_ATOMIC_BUILTINS
 763         mutex_create(rw_lock_get_mutex(lock));
 764         mutex_set_level(rw_lock_get_mutex(lock), SYNC_NO_ORDER_CHECK);
 765
 766 @@ -108,8 +109,14 @@
 767         lock->mutex.cmutex_name = cmutex_name;
 768         lock->mutex.mutex_type = 1;
 769  #endif /* UNIV_DEBUG && !UNIV_HOTBACKUP */
 770 +#endif /* !HAVE_ATOMIC_BUILTINS */
 771
 772 -       rw_lock_set_waiters(lock, 0);
 773 +#ifdef HAVE_ATOMIC_BUILTINS
 774 +       lock->lock_word = RW_LOCK_BIAS;
 775 +#endif
 776 +       rw_lock_set_s_waiters(lock, 0);
 777 +       rw_lock_set_x_waiters(lock, 0);
 778 +       rw_lock_set_wx_waiters(lock, 0);
 779         rw_lock_set_writer(lock, RW_LOCK_NOT_LOCKED);
 780         lock->writer_count = 0;
 781         rw_lock_set_reader_count(lock, 0);
 782 @@ -130,11 +137,9 @@
 783         lock->last_x_file_name = "not yet reserved";
 784         lock->last_s_line = 0;
 785         lock->last_x_line = 0;
 786 -       lock->event = os_event_create(NULL);
 787 -
 788 -#ifdef __WIN__
 789 +       lock->s_event = os_event_create(NULL);
 790 +       lock->x_event = os_event_create(NULL);
 791         lock->wait_ex_event = os_event_create(NULL);
 792 -#endif
 793
 794         mutex_enter(&rw_lock_list_mutex);
 795
 796 @@ -162,19 +167,21 @@
 797         ut_a(rw_lock_validate(lock));
 798  #endif /* UNIV_DEBUG */
 799         ut_a(rw_lock_get_writer(lock) == RW_LOCK_NOT_LOCKED);
 800 -       ut_a(rw_lock_get_waiters(lock) == 0);
 801 +       ut_a(rw_lock_get_s_waiters(lock) == 0);
 802 +       ut_a(rw_lock_get_x_waiters(lock) == 0);
 803 +       ut_a(rw_lock_get_wx_waiters(lock) == 0);
 804         ut_a(rw_lock_get_reader_count(lock) == 0);
 805
 806         lock->magic_n = 0;
 807
 808 +#ifndef HAVE_ATOMIC_BUILTINS
 809         mutex_free(rw_lock_get_mutex(lock));
 810 +#endif
 811
 812         mutex_enter(&rw_lock_list_mutex);
 813 -       os_event_free(lock->event);
 814 -
 815 -#ifdef __WIN__
 816 +       os_event_free(lock->s_event);
 817 +       os_event_free(lock->x_event);
 818         os_event_free(lock->wait_ex_event);
 819 -#endif
 820
 821         if (UT_LIST_GET_PREV(list, lock)) {
 822                 ut_a(UT_LIST_GET_PREV(list, lock)->magic_n == RW_LOCK_MAGIC_N);
 823 @@ -192,26 +199,43 @@
 824  Checks that the rw-lock has been initialized and that there are no
 825  simultaneous shared and exclusive locks. */
 826
 827 +/* MEMO: If HAVE_ATOMIC_BUILTINS, we should use this function statically. */
 828 +
 829  ibool
 830  rw_lock_validate(
 831  /*=============*/
 832         rw_lock_t*      lock)
 833  {
 834 +       ulint   test;
 835         ut_a(lock);
 836
 837 +#ifndef HAVE_ATOMIC_BUILTINS
 838         mutex_enter(rw_lock_get_mutex(lock));
 839 +#endif
 840
 841         ut_a(lock->magic_n == RW_LOCK_MAGIC_N);
 842 +#ifndef HAVE_ATOMIC_BUILTINS
 843         ut_a((rw_lock_get_reader_count(lock) == 0)
 844              || (rw_lock_get_writer(lock) != RW_LOCK_EX));
 845 -       ut_a((rw_lock_get_writer(lock) == RW_LOCK_EX)
 846 -            || (rw_lock_get_writer(lock) == RW_LOCK_WAIT_EX)
 847 -            || (rw_lock_get_writer(lock) == RW_LOCK_NOT_LOCKED));
 848 -       ut_a((rw_lock_get_waiters(lock) == 0)
 849 -            || (rw_lock_get_waiters(lock) == 1));
 850 +#endif
 851 +       test = rw_lock_get_writer(lock);
 852 +       ut_a((test == RW_LOCK_EX)
 853 +            || (test == RW_LOCK_WAIT_EX)
 854 +            || (test == RW_LOCK_NOT_LOCKED));
 855 +       test = rw_lock_get_s_waiters(lock);
 856 +       ut_a((test == 0)
 857 +            || (test == 1));
 858 +       test = rw_lock_get_x_waiters(lock);
 859 +       ut_a((test == 0)
 860 +            || (test == 1));
 861 +       test = rw_lock_get_wx_waiters(lock);
 862 +       ut_a((test == 0)
 863 +            || (test == 1));
 864 +#ifndef HAVE_ATOMIC_BUILTINS
 865         ut_a((lock->writer != RW_LOCK_EX) || (lock->writer_count > 0));
 866
 867         mutex_exit(rw_lock_get_mutex(lock));
 868 +#endif
 869
 870         return(TRUE);
 871  }
 872 @@ -237,13 +261,14 @@
 873          ut_ad(rw_lock_validate(lock));
 874
 875  lock_loop:
 876 +        i = 0;
 877 +spin_loop:
 878         rw_s_spin_wait_count++;
 879
 880         /* Spin waiting for the writer field to become free */
 881 -        i = 0;
 882
 883 -        while (rw_lock_get_writer(lock) != RW_LOCK_NOT_LOCKED
 884 -                                               && i < SYNC_SPIN_ROUNDS) {
 885 +        while (i < SYNC_SPIN_ROUNDS
 886 +                       && rw_lock_get_writer(lock) != RW_LOCK_NOT_LOCKED) {
 887                 if (srv_spin_wait_delay) {
 888                         ut_delay(ut_rnd_interval(0, srv_spin_wait_delay));
 889                 }
 890 @@ -262,15 +287,27 @@
 891                 lock->cfile_name, (ulong) lock->cline, (ulong) i);
 892         }
 893
 894 +#ifndef HAVE_ATOMIC_BUILTINS
 895         mutex_enter(rw_lock_get_mutex(lock));
 896 +#endif
 897
 898          /* We try once again to obtain the lock */
 899
 900         if (TRUE == rw_lock_s_lock_low(lock, pass, file_name, line)) {
 901 +#ifndef HAVE_ATOMIC_BUILTINS
 902                 mutex_exit(rw_lock_get_mutex(lock));
 903 +#endif
 904
 905                 return; /* Success */
 906         } else {
 907 +#ifdef HAVE_ATOMIC_BUILTINS
 908 +               /* like sync0sync.c doing */
 909 +               i++;
 910 +
 911 +               if (i < SYNC_SPIN_ROUNDS) {
 912 +                       goto spin_loop;
 913 +               }
 914 +#endif
 915                 /* If we get here, locking did not succeed, we may
 916                 suspend the thread to wait in the wait array */
 917
 918 @@ -281,9 +318,26 @@
 919                                 file_name, line,
 920                                 &index);
 921
 922 -               rw_lock_set_waiters(lock, 1);
 923 +               rw_lock_set_s_waiters(lock, 1);
 924 +
 925 +#ifdef HAVE_ATOMIC_BUILTINS
 926 +               /* like sync0sync.c doing */
 927 +               for (i = 0; i < 4; i++) {
 928 +                       if (TRUE == rw_lock_s_lock_low(lock, pass, file_name, line)) {
 929 +                               sync_array_free_cell(sync_primary_wait_array, index);
 930 +                               return; /* Success */
 931 +                       }
 932 +               }
 933
 934 +               /* If wait_ex_waiter stalls, wakes it. */
 935 +               if (lock->reader_count == 0
 936 +                   && __sync_lock_test_and_set(&lock->wait_ex_waiters, 0)) {
 937 +                       os_event_set(lock->wait_ex_event);
 938 +                       sync_array_object_signalled(sync_primary_wait_array);
 939 +               }
 940 +#else
 941                 mutex_exit(rw_lock_get_mutex(lock));
 942 +#endif
 943
 944                 if (srv_print_latch_waits) {
 945                         fprintf(stderr,
 946 @@ -318,13 +372,19 @@
 947  {
 948         ut_ad(rw_lock_is_locked(lock, RW_LOCK_EX));
 949
 950 +#ifndef HAVE_ATOMIC_BUILTINS
 951         mutex_enter(&(lock->mutex));
 952 +#endif
 953
 954         lock->writer_thread = os_thread_get_curr_id();
 955
 956         lock->pass = 0;
 957
 958 +#ifndef HAVE_ATOMIC_BUILTINS
 959         mutex_exit(&(lock->mutex));
 960 +#else
 961 +       __sync_synchronize();
 962 +#endif
 963  }
 964
 965  /**********************************************************************
 966 @@ -342,6 +402,89 @@
 967         const char*     file_name,/* in: file name where lock requested */
 968         ulint           line)   /* in: line where requested */
 969  {
 970 +#ifdef HAVE_ATOMIC_BUILTINS
 971 +       os_thread_id_t  curr_thread     = os_thread_get_curr_id();
 972 +retry_writer:
 973 +       /* try to lock writer */
 974 +       if(__sync_lock_test_and_set(&(lock->writer),RW_LOCK_EX)
 975 +                       == RW_LOCK_NOT_LOCKED) {
 976 +               /* success */
 977 +               /* obtain RW_LOCK_WAIT_EX right */
 978 +               lock->writer_thread = curr_thread;
 979 +               lock->pass = pass;
 980 +               lock->writer_is_wait_ex = TRUE;
 981 +               /* atomic operation may be safer about memory order. */
 982 +               __sync_synchronize();
 983 +#ifdef UNIV_SYNC_DEBUG
 984 +               rw_lock_add_debug_info(lock, pass, RW_LOCK_WAIT_EX,
 985 +                                       file_name, line);
 986 +#endif
 987 +       }
 988 +
 989 +       if (!os_thread_eq(lock->writer_thread, curr_thread)) {
 990 +               return(RW_LOCK_NOT_LOCKED);
 991 +       }
 992 +
 993 +       switch(rw_lock_get_writer(lock)) {
 994 +           case RW_LOCK_WAIT_EX:
 995 +               /* have right to try x-lock */
 996 +retry_x_lock:
 997 +               /* try x-lock */
 998 +               if(__sync_sub_and_fetch(&(lock->lock_word),
 999 +                               RW_LOCK_BIAS) == 0) {
1000 +                       /* success */
1001 +                       lock->pass = pass;
1002 +                       lock->writer_is_wait_ex = FALSE;
1003 +                       __sync_fetch_and_add(&(lock->writer_count),1);
1004 +
1005 +#ifdef UNIV_SYNC_DEBUG
1006 +                       rw_lock_remove_debug_info(lock, pass, RW_LOCK_WAIT_EX);
1007 +                       rw_lock_add_debug_info(lock, pass, RW_LOCK_EX,
1008 +                                               file_name, line);
1009 +#endif
1010 +
1011 +                       lock->last_x_file_name = file_name;
1012 +                       lock->last_x_line = line;
1013 +
1014 +                       /* Locking succeeded, we may return */
1015 +                       return(RW_LOCK_EX);
1016 +               } else if(__sync_fetch_and_add(&(lock->lock_word),
1017 +                               RW_LOCK_BIAS) == 0) {
1018 +                       /* retry x-lock */
1019 +                       goto retry_x_lock;
1020 +               }
1021 +
1022 +               /* There are readers, we have to wait */
1023 +               return(RW_LOCK_WAIT_EX);
1024 +
1025 +               break;
1026 +
1027 +           case RW_LOCK_EX:
1028 +               /* already have x-lock */
1029 +               if ((lock->pass == 0)&&(pass == 0)) {
1030 +                       __sync_fetch_and_add(&(lock->writer_count),1);
1031 +
1032 +#ifdef UNIV_SYNC_DEBUG
1033 +                       rw_lock_add_debug_info(lock, pass, RW_LOCK_EX, file_name,
1034 +                                               line);
1035 +#endif
1036 +
1037 +                       lock->last_x_file_name = file_name;
1038 +                       lock->last_x_line = line;
1039 +
1040 +                       /* Locking succeeded, we may return */
1041 +                       return(RW_LOCK_EX);
1042 +               }
1043 +
1044 +               return(RW_LOCK_NOT_LOCKED);
1045 +
1046 +               break;
1047 +
1048 +           default: /* RW_LOCK_NOT_LOCKED? maybe impossible */
1049 +               goto retry_writer;
1050 +       }
1051 +#else /* HAVE_ATOMIC_BUILTINS */
1052 +
1053  #ifdef UNIV_SYNC_DEBUG
1054         ut_ad(mutex_own(rw_lock_get_mutex(lock)));
1055  #endif /* UNIV_SYNC_DEBUG */
1056 @@ -423,6 +566,7 @@
1057                 /* Locking succeeded, we may return */
1058                 return(RW_LOCK_EX);
1059         }
1060 +#endif /* HAVE_ATOMIC_BUILTINS */
1061
1062         /* Locking did not succeed */
1063         return(RW_LOCK_NOT_LOCKED);
1064 @@ -448,19 +592,33 @@
1065         ulint           line)   /* in: line where requested */
1066  {
1067          ulint  index;  /* index of the reserved wait cell */
1068 -        ulint  state;  /* lock state acquired */
1069 +        ulint  state = RW_LOCK_NOT_LOCKED;     /* lock state acquired */
1070 +#ifdef HAVE_ATOMIC_BUILTINS
1071 +       ulint   prev_state = RW_LOCK_NOT_LOCKED;
1072 +#endif
1073          ulint  i;      /* spin round count */
1074
1075          ut_ad(rw_lock_validate(lock));
1076
1077  lock_loop:
1078 +       i = 0;
1079 +
1080 +#ifdef HAVE_ATOMIC_BUILTINS
1081 +       prev_state = state;
1082 +#else
1083          /* Acquire the mutex protecting the rw-lock fields */
1084         mutex_enter_fast(&(lock->mutex));
1085 +#endif
1086
1087         state = rw_lock_x_lock_low(lock, pass, file_name, line);
1088
1089 +#ifdef HAVE_ATOMIC_BUILTINS
1090 +       if (state != prev_state) i=0; /* if progress, reset counter. */
1091 +#else
1092         mutex_exit(&(lock->mutex));
1093 +#endif
1094
1095 +spin_loop:
1096         if (state == RW_LOCK_EX) {
1097
1098                 return; /* Locking succeeded */
1099 @@ -468,10 +626,9 @@
1100         } else if (state == RW_LOCK_NOT_LOCKED) {
1101
1102                 /* Spin waiting for the writer field to become free */
1103 -               i = 0;
1104
1105 -               while (rw_lock_get_writer(lock) != RW_LOCK_NOT_LOCKED
1106 -                                                       && i < SYNC_SPIN_ROUNDS) {
1107 +               while (i < SYNC_SPIN_ROUNDS
1108 +                       && lock->lock_word != RW_LOCK_BIAS) {
1109                         if (srv_spin_wait_delay) {
1110                                 ut_delay(ut_rnd_interval(0,
1111                                                         srv_spin_wait_delay));
1112 @@ -485,9 +642,12 @@
1113          } else if (state == RW_LOCK_WAIT_EX) {
1114
1115                 /* Spin waiting for the reader count field to become zero */
1116 -               i = 0;
1117
1118 +#ifdef HAVE_ATOMIC_BUILTINS
1119 +               while (lock->lock_word != RW_LOCK_BIAS
1120 +#else
1121                 while (rw_lock_get_reader_count(lock) != 0
1122 +#endif
1123                                                         && i < SYNC_SPIN_ROUNDS) {
1124                         if (srv_spin_wait_delay) {
1125                                 ut_delay(ut_rnd_interval(0,
1126 @@ -500,7 +660,6 @@
1127                         os_thread_yield();
1128                 }
1129          } else {
1130 -               i = 0; /* Eliminate a compiler warning */
1131                 ut_error;
1132         }
1133
1134 @@ -516,34 +675,69 @@
1135          /* We try once again to obtain the lock. Acquire the mutex protecting
1136         the rw-lock fields */
1137
1138 +#ifdef HAVE_ATOMIC_BUILTINS
1139 +       prev_state = state;
1140 +#else
1141         mutex_enter(rw_lock_get_mutex(lock));
1142 +#endif
1143
1144         state = rw_lock_x_lock_low(lock, pass, file_name, line);
1145
1146 +#ifdef HAVE_ATOMIC_BUILTINS
1147 +       if (state != prev_state) i=0; /* if progress, reset counter. */
1148 +#endif
1149 +
1150         if (state == RW_LOCK_EX) {
1151 +#ifndef HAVE_ATOMIC_BUILTINS
1152                 mutex_exit(rw_lock_get_mutex(lock));
1153 +#endif
1154
1155                 return; /* Locking succeeded */
1156         }
1157
1158 +#ifdef HAVE_ATOMIC_BUILTINS
1159 +       /* like sync0sync.c doing */
1160 +       i++;
1161 +
1162 +       if (i < SYNC_SPIN_ROUNDS) {
1163 +               goto spin_loop;
1164 +       }
1165 +#endif
1166 +
1167         rw_x_system_call_count++;
1168
1169          sync_array_reserve_cell(sync_primary_wait_array,
1170                                 lock,
1171 -#ifdef __WIN__
1172 -                               /* On windows RW_LOCK_WAIT_EX signifies
1173 -                               that this thread should wait on the
1174 -                               special wait_ex_event. */
1175                                 (state == RW_LOCK_WAIT_EX)
1176                                  ? RW_LOCK_WAIT_EX :
1177 -#endif
1178                                 RW_LOCK_EX,
1179                                 file_name, line,
1180                                 &index);
1181
1182 -       rw_lock_set_waiters(lock, 1);
1183 +       if (state == RW_LOCK_WAIT_EX) {
1184 +               rw_lock_set_wx_waiters(lock, 1);
1185 +       } else {
1186 +               rw_lock_set_x_waiters(lock, 1);
1187 +       }
1188
1189 +#ifdef HAVE_ATOMIC_BUILTINS
1190 +       /* like sync0sync.c doing */
1191 +       for (i = 0; i < 4; i++) {
1192 +               prev_state = state;
1193 +               state = rw_lock_x_lock_low(lock, pass, file_name, line);
1194 +               if (state == RW_LOCK_EX) {
1195 +                       sync_array_free_cell(sync_primary_wait_array, index);
1196 +                       return; /* Locking succeeded */
1197 +               }
1198 +               if (state != prev_state) {
1199 +                       /* retry! */
1200 +                       sync_array_free_cell(sync_primary_wait_array, index);
1201 +                       goto lock_loop;
1202 +               }
1203 +       }
1204 +#else
1205         mutex_exit(rw_lock_get_mutex(lock));
1206 +#endif
1207
1208         if (srv_print_latch_waits) {
1209                 fprintf(stderr,
1210 @@ -718,7 +912,9 @@
1211         ut_ad(lock);
1212         ut_ad(rw_lock_validate(lock));
1213
1214 +#ifndef HAVE_ATOMIC_BUILTINS
1215         mutex_enter(&(lock->mutex));
1216 +#endif
1217
1218         info = UT_LIST_GET_FIRST(lock->debug_list);
1219
1220 @@ -728,7 +924,9 @@
1221                     && (info->pass == 0)
1222                     && (info->lock_type == lock_type)) {
1223
1224 +#ifndef HAVE_ATOMIC_BUILTINS
1225                         mutex_exit(&(lock->mutex));
1226 +#endif
1227                         /* Found! */
1228
1229                         return(TRUE);
1230 @@ -736,7 +934,9 @@
1231
1232                 info = UT_LIST_GET_NEXT(list, info);
1233         }
1234 +#ifndef HAVE_ATOMIC_BUILTINS
1235         mutex_exit(&(lock->mutex));
1236 +#endif
1237
1238         return(FALSE);
1239  }
1240 @@ -758,21 +958,25 @@
1241         ut_ad(lock);
1242         ut_ad(rw_lock_validate(lock));
1243
1244 +#ifndef HAVE_ATOMIC_BUILTINS
1245         mutex_enter(&(lock->mutex));
1246 +#endif
1247
1248         if (lock_type == RW_LOCK_SHARED) {
1249                 if (lock->reader_count > 0) {
1250                         ret = TRUE;
1251                 }
1252         } else if (lock_type == RW_LOCK_EX) {
1253 -               if (lock->writer == RW_LOCK_EX) {
1254 +               if (rw_lock_get_writer(lock) == RW_LOCK_EX) {
1255                         ret = TRUE;
1256                 }
1257         } else {
1258                 ut_error;
1259         }
1260
1261 +#ifndef HAVE_ATOMIC_BUILTINS
1262         mutex_exit(&(lock->mutex));
1263 +#endif
1264
1265         return(ret);
1266  }
1267 @@ -801,16 +1005,26 @@
1268
1269                 count++;
1270
1271 +#ifndef HAVE_ATOMIC_BUILTINS
1272                 mutex_enter(&(lock->mutex));
1273 +#endif
1274
1275                 if ((rw_lock_get_writer(lock) != RW_LOCK_NOT_LOCKED)
1276                     || (rw_lock_get_reader_count(lock) != 0)
1277 -                   || (rw_lock_get_waiters(lock) != 0)) {
1278 +                   || (rw_lock_get_s_waiters(lock) != 0)
1279 +                   || (rw_lock_get_x_waiters(lock) != 0)
1280 +                   || (rw_lock_get_wx_waiters(lock) != 0)) {
1281
1282                         fprintf(stderr, "RW-LOCK: %p ", lock);
1283
1284 -                       if (rw_lock_get_waiters(lock)) {
1285 -                               fputs(" Waiters for the lock exist\n", stderr);
1286 +                       if (rw_lock_get_s_waiters(lock)) {
1287 +                               fputs(" s_waiters for the lock exist,", stderr);
1288 +                       }
1289 +                       if (rw_lock_get_x_waiters(lock)) {
1290 +                               fputs(" x_waiters for the lock exist\n", stderr);
1291 +                       }
1292 +                       if (rw_lock_get_wx_waiters(lock)) {
1293 +                               fputs(" wait_ex_waiters for the lock exist\n", stderr);
1294                         } else {
1295                                 putc('\n', stderr);
1296                         }
1297 @@ -822,7 +1036,9 @@
1298                         }
1299                 }
1300
1301 +#ifndef HAVE_ATOMIC_BUILTINS
1302                 mutex_exit(&(lock->mutex));
1303 +#endif
1304                 lock = UT_LIST_GET_NEXT(list, lock);
1305         }
1306
1307 @@ -847,10 +1063,18 @@
1308
1309         if ((rw_lock_get_writer(lock) != RW_LOCK_NOT_LOCKED)
1310             || (rw_lock_get_reader_count(lock) != 0)
1311 -           || (rw_lock_get_waiters(lock) != 0)) {
1312 +           || (rw_lock_get_s_waiters(lock) != 0)
1313 +           || (rw_lock_get_x_waiters(lock) != 0)
1314 +           || (rw_lock_get_wx_waiters(lock) != 0)) {
1315
1316 -               if (rw_lock_get_waiters(lock)) {
1317 -                       fputs(" Waiters for the lock exist\n", stderr);
1318 +               if (rw_lock_get_s_waiters(lock)) {
1319 +                       fputs(" s_waiters for the lock exist,", stderr);
1320 +               }
1321 +               if (rw_lock_get_x_waiters(lock)) {
1322 +                       fputs(" x_waiters for the lock exist\n", stderr);
1323 +               }
1324 +               if (rw_lock_get_wx_waiters(lock)) {
1325 +                       fputs(" wait_ex_waiters for the lock exist\n", stderr);
1326                 } else {
1327                         putc('\n', stderr);
1328                 }
1329 @@ -909,14 +1133,18 @@
1330         lock = UT_LIST_GET_FIRST(rw_lock_list);
1331
1332         while (lock != NULL) {
1333 +#ifndef HAVE_ATOMIC_BUILTINS
1334                 mutex_enter(rw_lock_get_mutex(lock));
1335 +#endif
1336
1337                 if ((rw_lock_get_writer(lock) != RW_LOCK_NOT_LOCKED)
1338                                 || (rw_lock_get_reader_count(lock) != 0)) {
1339                         count++;
1340                 }
1341
1342 +#ifndef HAVE_ATOMIC_BUILTINS
1343                 mutex_exit(rw_lock_get_mutex(lock));
1344 +#endif
1345                 lock = UT_LIST_GET_NEXT(list, lock);
1346         }
1347
1348 diff -ruN a/patch_info/innodb_rw_lock.info b/patch_info/innodb_rw_lock.info
1349 --- /dev/null   1970-01-01 09:00:00.000000000 +0900
1350 +++ b/patch_info/innodb_rw_lock.info    2009-04-16 16:15:28.000000000 +0900
1351 @@ -0,0 +1,6 @@
1352 +File=innodb_rw_lock.patch
1353 +Name=Fix of InnoDB rw_locks
1354 +Version=1.0
1355 +Author=Yasufumi Kinoshita
1356 +License=BSD
1357 +Comment=