innodb_recovery_patches.patch

   1 # name       : innodb_recovery_patches.patch
   2 # introduced : 11 or before
   3 # maintainer : Yasufumi
   4 #
   5 #!!! notice !!!
   6 # Any small change to this file in the main branch
   7 # should be done or reviewed by the maintainer!
   8 diff -ruN a/storage/innobase/buf/buf0rea.c b/storage/innobase/buf/buf0rea.c
   9 --- a/storage/innobase/buf/buf0rea.c    2010-12-03 15:49:59.187028943 +0900
  10 +++ b/storage/innobase/buf/buf0rea.c    2010-12-03 17:30:41.579956150 +0900
  11 @@ -122,6 +122,46 @@
  12         bpage = buf_page_init_for_read(err, mode, space, zip_size, unzip,
  13                                        tablespace_version, offset);
  14         if (bpage == NULL) {
  15 +               /* bugfix: http://bugs.mysql.com/bug.php?id=43948 */
  16 +               if (recv_recovery_is_on() && *err == DB_TABLESPACE_DELETED) {
  17 +                       /* hashed log recs must be treated here */
  18 +                       recv_addr_t*    recv_addr;
  19 +
  20 +                       mutex_enter(&(recv_sys->mutex));
  21 +
  22 +                       if (recv_sys->apply_log_recs == FALSE) {
  23 +                               mutex_exit(&(recv_sys->mutex));
  24 +                               goto not_to_recover;
  25 +                       }
  26 +
  27 +                       /* recv_get_fil_addr_struct() */
  28 +                       recv_addr = HASH_GET_FIRST(recv_sys->addr_hash,
  29 +                                       hash_calc_hash(ut_fold_ulint_pair(space, offset),
  30 +                                               recv_sys->addr_hash));
  31 +                       while (recv_addr) {
  32 +                               if ((recv_addr->space == space)
  33 +                                       && (recv_addr->page_no == offset)) {
  34 +                                       break;
  35 +                               }
  36 +                               recv_addr = HASH_GET_NEXT(addr_hash, recv_addr);
  37 +                       }
  38 +
  39 +                       if ((recv_addr == NULL)
  40 +                           || (recv_addr->state == RECV_BEING_PROCESSED)
  41 +                           || (recv_addr->state == RECV_PROCESSED)) {
  42 +                               mutex_exit(&(recv_sys->mutex));
  43 +                               goto not_to_recover;
  44 +                       }
  45 +
  46 +                       fprintf(stderr, " (cannot find space: %lu)", space);
  47 +                       recv_addr->state = RECV_PROCESSED;
  48 +
  49 +                       ut_a(recv_sys->n_addrs);
  50 +                       recv_sys->n_addrs--;
  51 +
  52 +                       mutex_exit(&(recv_sys->mutex));
  53 +               }
  54 +not_to_recover:
  55
  56                 return(0);
  57         }
  58 @@ -613,6 +653,50 @@
  59                 /* It is a single table tablespace and the .ibd file is
  60                 missing: do nothing */
  61
  62 +               /* the log records should be treated here same reason
  63 +               for http://bugs.mysql.com/bug.php?id=43948 */
  64 +
  65 +               if (recv_recovery_is_on()) {
  66 +                       recv_addr_t*    recv_addr;
  67 +
  68 +                       mutex_enter(&(recv_sys->mutex));
  69 +
  70 +                       if (recv_sys->apply_log_recs == FALSE) {
  71 +                               mutex_exit(&(recv_sys->mutex));
  72 +                               goto not_to_recover;
  73 +                       }
  74 +
  75 +                       for (i = 0; i < n_stored; i++) {
  76 +                               /* recv_get_fil_addr_struct() */
  77 +                               recv_addr = HASH_GET_FIRST(recv_sys->addr_hash,
  78 +                                               hash_calc_hash(ut_fold_ulint_pair(space, page_nos[i]),
  79 +                                                       recv_sys->addr_hash));
  80 +                               while (recv_addr) {
  81 +                                       if ((recv_addr->space == space)
  82 +                                               && (recv_addr->page_no == page_nos[i])) {
  83 +                                               break;
  84 +                                       }
  85 +                                       recv_addr = HASH_GET_NEXT(addr_hash, recv_addr);
  86 +                               }
  87 +
  88 +                               if ((recv_addr == NULL)
  89 +                                   || (recv_addr->state == RECV_BEING_PROCESSED)
  90 +                                   || (recv_addr->state == RECV_PROCESSED)) {
  91 +                                       continue;
  92 +                               }
  93 +
  94 +                               recv_addr->state = RECV_PROCESSED;
  95 +
  96 +                               ut_a(recv_sys->n_addrs);
  97 +                               recv_sys->n_addrs--;
  98 +                       }
  99 +
 100 +                       mutex_exit(&(recv_sys->mutex));
 101 +
 102 +                       fprintf(stderr, " (cannot find space: %lu)", space);
 103 +               }
 104 +not_to_recover:
 105 +
 106                 return;
 107         }
 108
 109 diff -ruN a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc
 110 --- a/storage/innobase/handler/ha_innodb.cc     2010-12-03 17:30:16.261955714 +0900
 111 +++ b/storage/innobase/handler/ha_innodb.cc     2010-12-03 17:30:41.584971130 +0900
 112 @@ -182,6 +182,7 @@
 113  #endif /* UNIV_LOG_ARCHIVE */
 114  static my_bool innobase_use_doublewrite                = TRUE;
 115  static my_bool innobase_use_checksums                  = TRUE;
 116 +static my_bool innobase_recovery_stats                 = TRUE;
 117  static my_bool innobase_locks_unsafe_for_binlog        = FALSE;
 118  static my_bool innobase_overwrite_relay_log_info       = FALSE;
 119  static my_bool innobase_rollback_on_timeout            = FALSE;
 120 @@ -2549,6 +2550,8 @@
 121
 122         srv_force_recovery = (ulint) innobase_force_recovery;
 123
 124 +       srv_recovery_stats = (ibool) innobase_recovery_stats;
 125 +
 126         srv_use_doublewrite_buf = (ibool) innobase_use_doublewrite;
 127         srv_use_checksums = (ibool) innobase_use_checksums;
 128
 129 @@ -11250,6 +11253,11 @@
 130    "The common part for InnoDB table spaces.",
 131    NULL, NULL, NULL);
 132
 133 +static MYSQL_SYSVAR_BOOL(recovery_stats, innobase_recovery_stats,
 134 +  PLUGIN_VAR_NOCMDARG | PLUGIN_VAR_READONLY,
 135 +  "Output statistics of recovery process after it.",
 136 +  NULL, NULL, FALSE);
 137 +
 138  static MYSQL_SYSVAR_BOOL(recovery_update_relay_log, innobase_overwrite_relay_log_info,
 139    PLUGIN_VAR_NOCMDARG | PLUGIN_VAR_READONLY,
 140    "During InnoDB crash recovery on slave overwrite relay-log.info "
 141 @@ -11723,6 +11731,7 @@
 142    MYSQL_SYSVAR(data_file_path),
 143    MYSQL_SYSVAR(data_home_dir),
 144    MYSQL_SYSVAR(doublewrite),
 145 +  MYSQL_SYSVAR(recovery_stats),
 146    MYSQL_SYSVAR(fast_shutdown),
 147    MYSQL_SYSVAR(file_io_threads),
 148    MYSQL_SYSVAR(read_io_threads),
 149 diff -ruN a/storage/innobase/include/log0recv.h b/storage/innobase/include/log0recv.h
 150 --- a/storage/innobase/include/log0recv.h       2010-11-03 07:01:13.000000000 +0900
 151 +++ b/storage/innobase/include/log0recv.h       2010-12-03 17:30:41.592958318 +0900
 152 @@ -438,6 +438,39 @@
 153         hash_table_t*   addr_hash;/*!< hash table of file addresses of pages */
 154         ulint           n_addrs;/*!< number of not processed hashed file
 155                                 addresses in the hash table */
 156 +
 157 +/* If you modified the following defines at original file,
 158 +   You should also modify them. */
 159 +/* defined in os0file.c */
 160 +#define OS_AIO_MERGE_N_CONSECUTIVE     64
 161 +/* defined in log0recv.c */
 162 +#define RECV_READ_AHEAD_AREA   32
 163 +       time_t          stats_recv_start_time;
 164 +       ulint           stats_recv_turns;
 165 +
 166 +       ulint           stats_read_requested_pages;
 167 +       ulint           stats_read_in_area[RECV_READ_AHEAD_AREA];
 168 +
 169 +       ulint           stats_read_io_pages;
 170 +       ulint           stats_read_io_consecutive[OS_AIO_MERGE_N_CONSECUTIVE];
 171 +       ulint           stats_write_io_pages;
 172 +       ulint           stats_write_io_consecutive[OS_AIO_MERGE_N_CONSECUTIVE];
 173 +
 174 +       ulint           stats_doublewrite_check_pages;
 175 +       ulint           stats_doublewrite_overwrite_pages;
 176 +
 177 +       ulint           stats_recover_pages_with_read;
 178 +       ulint           stats_recover_pages_without_read;
 179 +
 180 +       ulint           stats_log_recs;
 181 +       ulint           stats_log_len_sum;
 182 +
 183 +       ulint           stats_applied_log_recs;
 184 +       ulint           stats_applied_log_len_sum;
 185 +       ulint           stats_pages_already_new;
 186 +
 187 +       ib_uint64_t     stats_oldest_modified_lsn;
 188 +       ib_uint64_t     stats_newest_modified_lsn;
 189  };
 190
 191  /** The recovery system */
 192 diff -ruN a/storage/innobase/include/srv0srv.h b/storage/innobase/include/srv0srv.h
 193 --- a/storage/innobase/include/srv0srv.h        2010-12-03 17:30:16.321953515 +0900
 194 +++ b/storage/innobase/include/srv0srv.h        2010-12-03 17:30:41.593985184 +0900
 195 @@ -129,6 +129,8 @@
 196  extern ulint*  srv_data_file_sizes;
 197  extern ulint*  srv_data_file_is_raw_partition;
 198
 199 +extern ibool   srv_recovery_stats;
 200 +
 201  extern ibool   srv_auto_extend_last_data_file;
 202  extern ulint   srv_last_file_size_max;
 203  extern char**  srv_log_group_home_dirs;
 204 diff -ruN a/storage/innobase/log/log0recv.c b/storage/innobase/log/log0recv.c
 205 --- a/storage/innobase/log/log0recv.c   2010-12-03 15:18:48.903987466 +0900
 206 +++ b/storage/innobase/log/log0recv.c   2010-12-03 17:30:41.598022536 +0900
 207 @@ -187,6 +187,9 @@
 208
 209         recv_sys->heap = NULL;
 210         recv_sys->addr_hash = NULL;
 211 +
 212 +       recv_sys->stats_recv_start_time = time(NULL);
 213 +       recv_sys->stats_oldest_modified_lsn = IB_ULONGLONG_MAX;
 214  }
 215
 216  /********************************************************//**
 217 @@ -327,6 +330,11 @@
 218                 recv_n_pool_free_frames = 512;
 219         }
 220
 221 +       if (buf_pool_get_curr_size() >= (32 * 1024 * 1024)) {
 222 +               /* Buffer pool of size greater than 32 MB. */
 223 +               recv_n_pool_free_frames = 1024;
 224 +       }
 225 +
 226         recv_sys->buf = ut_malloc(RECV_PARSING_BUF_SIZE);
 227         recv_sys->len = 0;
 228         recv_sys->recovered_offset = 0;
 229 @@ -1363,6 +1371,11 @@
 230
 231         len = rec_end - body;
 232
 233 +       if (srv_recovery_stats) {
 234 +               recv_sys->stats_log_recs++;
 235 +               recv_sys->stats_log_len_sum += len;
 236 +       }
 237 +
 238         recv = mem_heap_alloc(recv_sys->heap, sizeof(recv_t));
 239         recv->type = type;
 240         recv->len = rec_end - body;
 241 @@ -1474,6 +1487,7 @@
 242         ib_uint64_t     start_lsn;
 243         ib_uint64_t     end_lsn;
 244         ib_uint64_t     page_lsn;
 245 +       ib_uint64_t     page_lsn_orig;
 246         ib_uint64_t     page_newest_lsn;
 247         ibool           modification_to_page;
 248  #ifndef UNIV_HOTBACKUP
 249 @@ -1496,6 +1510,8 @@
 250                                              buf_block_get_page_no(block));
 251
 252         if ((recv_addr == NULL)
 253 +               /* bugfix: http://bugs.mysql.com/bug.php?id=44140 */
 254 +           || (recv_addr->state == RECV_BEING_READ && !just_read_in)
 255             || (recv_addr->state == RECV_BEING_PROCESSED)
 256             || (recv_addr->state == RECV_PROCESSED)) {
 257
 258 @@ -1511,6 +1527,14 @@
 259
 260         recv_addr->state = RECV_BEING_PROCESSED;
 261
 262 +       if (srv_recovery_stats) {
 263 +               if (just_read_in) {
 264 +                       recv_sys->stats_recover_pages_with_read++;
 265 +               } else {
 266 +                       recv_sys->stats_recover_pages_without_read++;
 267 +               }
 268 +       }
 269 +
 270         mutex_exit(&(recv_sys->mutex));
 271
 272         mtr_start(&mtr);
 273 @@ -1540,6 +1564,7 @@
 274
 275         /* Read the newest modification lsn from the page */
 276         page_lsn = mach_read_from_8(page + FIL_PAGE_LSN);
 277 +       page_lsn_orig = page_lsn;
 278
 279  #ifndef UNIV_HOTBACKUP
 280         /* It may be that the page has been modified in the buffer
 281 @@ -1559,6 +1584,21 @@
 282         modification_to_page = FALSE;
 283         start_lsn = end_lsn = 0;
 284
 285 +       if (srv_recovery_stats) {
 286 +               mutex_enter(&(recv_sys->mutex));
 287 +               if (page_lsn_orig && recv_sys->stats_oldest_modified_lsn > page_lsn_orig) {
 288 +                       recv_sys->stats_oldest_modified_lsn = page_lsn_orig;
 289 +               }
 290 +               if (page_lsn_orig && recv_sys->stats_newest_modified_lsn < page_lsn_orig) {
 291 +                       recv_sys->stats_newest_modified_lsn = page_lsn_orig;
 292 +               }
 293 +               if (UT_LIST_GET_LAST(recv_addr->rec_list)->start_lsn
 294 +                   < page_lsn_orig) {
 295 +                       recv_sys->stats_pages_already_new++;
 296 +               }
 297 +               mutex_exit(&(recv_sys->mutex));
 298 +       }
 299 +
 300         recv = UT_LIST_GET_FIRST(recv_addr->rec_list);
 301
 302         while (recv) {
 303 @@ -1613,6 +1653,13 @@
 304                                                          buf + recv->len,
 305                                                          block, &mtr);
 306
 307 +                       if (srv_recovery_stats) {
 308 +                               mutex_enter(&(recv_sys->mutex));
 309 +                               recv_sys->stats_applied_log_recs++;
 310 +                               recv_sys->stats_applied_log_len_sum += recv->len;
 311 +                               mutex_exit(&(recv_sys->mutex));
 312 +                       }
 313 +
 314                         end_lsn = recv->start_lsn + recv->len;
 315                         mach_write_to_8(FIL_PAGE_LSN + page, end_lsn);
 316                         mach_write_to_8(UNIV_PAGE_SIZE
 317 @@ -1715,6 +1762,13 @@
 318                 }
 319         }
 320
 321 +       if (srv_recovery_stats && n) {
 322 +               mutex_enter(&(recv_sys->mutex));
 323 +               recv_sys->stats_read_requested_pages += n;
 324 +               recv_sys->stats_read_in_area[n - 1]++;
 325 +               mutex_exit(&(recv_sys->mutex));
 326 +       }
 327 +
 328         buf_read_recv_pages(FALSE, space, zip_size, page_nos, n);
 329         /*
 330         fprintf(stderr, "Recv pages at %lu n %lu\n", page_nos[0], n);
 331 @@ -1867,6 +1921,10 @@
 332
 333         if (has_printed) {
 334                 fprintf(stderr, "InnoDB: Apply batch completed\n");
 335 +
 336 +               if (srv_recovery_stats) {
 337 +                       recv_sys->stats_recv_turns++;
 338 +               }
 339         }
 340
 341         mutex_exit(&(recv_sys->mutex));
 342 @@ -3270,6 +3328,90 @@
 343         }
 344  #endif /* UNIV_DEBUG */
 345
 346 +       if (recv_needed_recovery && srv_recovery_stats) {
 347 +               ulint   flush_list_len = 0;
 348 +               ulint   i;
 349 +
 350 +               fprintf(stderr,
 351 +                       "InnoDB: Log records have been applied. The statistics that were gathered follow.\n");
 352 +
 353 +               fprintf(stderr,
 354 +                       "============================================================\n"
 355 +                       "-------------------\n"
 356 +                       "RECOVERY STATISTICS\n"
 357 +                       "-------------------\n");
 358 +               fprintf(stderr,
 359 +                       "Recovery time: %g sec. (%lu turns)\n",
 360 +                       difftime(time(NULL), recv_sys->stats_recv_start_time),
 361 +                       recv_sys->stats_recv_turns);
 362 +
 363 +               for (i = 0; i < srv_buf_pool_instances; i++) {
 364 +                       buf_pool_t*     buf_pool;
 365 +
 366 +                       buf_pool = buf_pool_from_array(i);
 367 +                       flush_list_len += UT_LIST_GET_LEN(buf_pool->flush_list);
 368 +               }
 369 +               fprintf(stderr,
 370 +                       "\n"
 371 +                       "Data page IO statistics\n"
 372 +                       "  Requested pages: %lu\n"
 373 +                       "  Read pages:      %lu\n"
 374 +                       "  Written pages:   %lu\n"
 375 +                       "  (Dirty blocks):  %lu\n",
 376 +                       recv_sys->stats_read_requested_pages,
 377 +                       recv_sys->stats_read_io_pages,
 378 +                       recv_sys->stats_write_io_pages,
 379 +                       flush_list_len);
 380 +
 381 +               fprintf(stderr,
 382 +                       "  Grouping IO [times]:\n"
 383 +                       "\tnumber of pages,\n"
 384 +                       "\t\tread request neighbors (in %d pages chunk),\n"
 385 +                       "\t\t\tcombined read IO,\n"
 386 +                       "\t\t\t\tcombined write IO\n",
 387 +                       RECV_READ_AHEAD_AREA);
 388 +               for (i = 0; i < ut_max(RECV_READ_AHEAD_AREA,
 389 +                                       OS_AIO_MERGE_N_CONSECUTIVE); i++) {
 390 +                       fprintf(stderr,
 391 +                               "\t%3lu,\t%lu,\t%lu,\t%lu\n", i + 1,
 392 +                               (i < RECV_READ_AHEAD_AREA) ?
 393 +                                       recv_sys->stats_read_in_area[i] : 0,
 394 +                               (i < OS_AIO_MERGE_N_CONSECUTIVE) ?
 395 +                                       recv_sys->stats_read_io_consecutive[i] : 0,
 396 +                               (i < OS_AIO_MERGE_N_CONSECUTIVE) ?
 397 +                                       recv_sys->stats_write_io_consecutive[i] : 0);
 398 +               }
 399 +
 400 +               fprintf(stderr,
 401 +                       "\n"
 402 +                       "Recovery process statistics\n"
 403 +                       "  Checked pages by doublewrite buffer: %lu\n"
 404 +                       "  Overwritten pages from doublewrite:  %lu\n"
 405 +                       "  Recovered pages by io_thread:        %lu\n"
 406 +                       "  Recovered pages by main thread:      %lu\n"
 407 +                       "  Parsed log records to apply:         %lu\n"
 408 +                       "            Sum of the length:         %lu\n"
 409 +                       "  Applied log records:                 %lu\n"
 410 +                       "            Sum of the length:         %lu\n"
 411 +                       "  Pages which are already new enough:  %lu (It may not be accurate, if turns > 1)\n"
 412 +                       "  Oldest page's LSN:                   %llu\n"
 413 +                       "  Newest page's LSN:                   %llu\n",
 414 +                       recv_sys->stats_doublewrite_check_pages,
 415 +                       recv_sys->stats_doublewrite_overwrite_pages,
 416 +                       recv_sys->stats_recover_pages_with_read,
 417 +                       recv_sys->stats_recover_pages_without_read,
 418 +                       recv_sys->stats_log_recs,
 419 +                       recv_sys->stats_log_len_sum,
 420 +                       recv_sys->stats_applied_log_recs,
 421 +                       recv_sys->stats_applied_log_len_sum,
 422 +                       recv_sys->stats_pages_already_new,
 423 +                       recv_sys->stats_oldest_modified_lsn,
 424 +                       recv_sys->stats_newest_modified_lsn);
 425 +
 426 +               fprintf(stderr,
 427 +                       "============================================================\n");
 428 +       }
 429 +
 430         if (recv_needed_recovery) {
 431                 trx_sys_print_mysql_master_log_pos();
 432                 trx_sys_print_mysql_binlog_offset();
 433 diff -ruN a/storage/innobase/os/os0file.c b/storage/innobase/os/os0file.c
 434 --- a/storage/innobase/os/os0file.c     2010-12-03 15:18:48.908955759 +0900
 435 +++ b/storage/innobase/os/os0file.c     2010-12-03 17:30:41.602022989 +0900
 436 @@ -43,6 +43,7 @@
 437  #include "srv0start.h"
 438  #include "fil0fil.h"
 439  #include "buf0buf.h"
 440 +#include "log0recv.h"
 441  #ifndef UNIV_HOTBACKUP
 442  # include "os0sync.h"
 443  # include "os0thread.h"
 444 @@ -4262,6 +4263,18 @@
 445                 os_thread_exit(NULL);
 446         }
 447
 448 +       if (srv_recovery_stats && recv_recovery_is_on() && n_consecutive) {
 449 +               mutex_enter(&(recv_sys->mutex));
 450 +               if (slot->type == OS_FILE_READ) {
 451 +                       recv_sys->stats_read_io_pages += n_consecutive;
 452 +                       recv_sys->stats_read_io_consecutive[n_consecutive - 1]++;
 453 +               } else if (slot->type == OS_FILE_WRITE) {
 454 +                       recv_sys->stats_write_io_pages += n_consecutive;
 455 +                       recv_sys->stats_write_io_consecutive[n_consecutive - 1]++;
 456 +               }
 457 +               mutex_exit(&(recv_sys->mutex));
 458 +       }
 459 +
 460         os_mutex_enter(array->mutex);
 461
 462         slot = os_aio_array_get_nth_slot(array, i + segment * n);
 463 diff -ruN a/storage/innobase/srv/srv0srv.c b/storage/innobase/srv/srv0srv.c
 464 --- a/storage/innobase/srv/srv0srv.c    2010-12-03 17:30:16.339955597 +0900
 465 +++ b/storage/innobase/srv/srv0srv.c    2010-12-03 17:30:41.604958138 +0900
 466 @@ -167,6 +167,8 @@
 467  /* size in database pages */
 468  UNIV_INTERN ulint*     srv_data_file_sizes = NULL;
 469
 470 +UNIV_INTERN ibool      srv_recovery_stats = FALSE;
 471 +
 472  /* if TRUE, then we auto-extend the last data file */
 473  UNIV_INTERN ibool      srv_auto_extend_last_data_file  = FALSE;
 474  /* if != 0, this tells the max size auto-extending may increase the
 475 diff -ruN a/storage/innobase/trx/trx0sys.c b/storage/innobase/trx/trx0sys.c
 476 --- a/storage/innobase/trx/trx0sys.c    2010-12-03 15:41:52.051986524 +0900
 477 +++ b/storage/innobase/trx/trx0sys.c    2010-12-03 17:30:41.607026818 +0900
 478 @@ -566,6 +566,12 @@
 479                                zip_size ? zip_size : UNIV_PAGE_SIZE,
 480                                read_buf, NULL);
 481
 482 +                       if (srv_recovery_stats && recv_recovery_is_on()) {
 483 +                               mutex_enter(&(recv_sys->mutex));
 484 +                               recv_sys->stats_doublewrite_check_pages++;
 485 +                               mutex_exit(&(recv_sys->mutex));
 486 +                       }
 487 +
 488                         /* Check if the page is corrupt */
 489
 490                         if (UNIV_UNLIKELY
 491 @@ -613,6 +619,13 @@
 492                                        zip_size, page_no, 0,
 493                                        zip_size ? zip_size : UNIV_PAGE_SIZE,
 494                                        page, NULL);
 495 +
 496 +                               if (srv_recovery_stats && recv_recovery_is_on()) {
 497 +                                       mutex_enter(&(recv_sys->mutex));
 498 +                                       recv_sys->stats_doublewrite_overwrite_pages++;
 499 +                                       mutex_exit(&(recv_sys->mutex));
 500 +                               }
 501 +
 502                                 fprintf(stderr,
 503                                         "InnoDB: Recovered the page from"
 504                                         " the doublewrite buffer.\n");