innodb_recovery_patches.patch

   1 # name       : innodb_recovery_patches.patch
   2 # introduced : 11 or before
   3 # maintainer : Yasufumi
   4 #
   5 #!!! notice !!!
   6 # Any small change to this file in the main branch
   7 # should be done or reviewed by the maintainer!
   8 --- a/storage/innobase/buf/buf0rea.c
   9 +++ b/storage/innobase/buf/buf0rea.c
  10 @@ -124,6 +124,46 @@
  11         bpage = buf_page_init_for_read(err, mode, space, zip_size, unzip,
  12                                        tablespace_version, offset);
  13         if (bpage == NULL) {
  14 +               /* bugfix: http://bugs.mysql.com/bug.php?id=43948 */
  15 +               if (recv_recovery_is_on() && *err == DB_TABLESPACE_DELETED) {
  16 +                       /* hashed log recs must be treated here */
  17 +                       recv_addr_t*    recv_addr;
  18 +
  19 +                       mutex_enter(&(recv_sys->mutex));
  20 +
  21 +                       if (recv_sys->apply_log_recs == FALSE) {
  22 +                               mutex_exit(&(recv_sys->mutex));
  23 +                               goto not_to_recover;
  24 +                       }
  25 +
  26 +                       /* recv_get_fil_addr_struct() */
  27 +                       recv_addr = HASH_GET_FIRST(recv_sys->addr_hash,
  28 +                                       hash_calc_hash(ut_fold_ulint_pair(space, offset),
  29 +                                               recv_sys->addr_hash));
  30 +                       while (recv_addr) {
  31 +                               if ((recv_addr->space == space)
  32 +                                       && (recv_addr->page_no == offset)) {
  33 +                                       break;
  34 +                               }
  35 +                               recv_addr = HASH_GET_NEXT(addr_hash, recv_addr);
  36 +                       }
  37 +
  38 +                       if ((recv_addr == NULL)
  39 +                           || (recv_addr->state == RECV_BEING_PROCESSED)
  40 +                           || (recv_addr->state == RECV_PROCESSED)) {
  41 +                               mutex_exit(&(recv_sys->mutex));
  42 +                               goto not_to_recover;
  43 +                       }
  44 +
  45 +                       fprintf(stderr, " (cannot find space: %lu)", space);
  46 +                       recv_addr->state = RECV_PROCESSED;
  47 +
  48 +                       ut_a(recv_sys->n_addrs);
  49 +                       recv_sys->n_addrs--;
  50 +
  51 +                       mutex_exit(&(recv_sys->mutex));
  52 +               }
  53 +not_to_recover:
  54
  55                 return(0);
  56         }
  57 @@ -777,6 +817,50 @@
  58                 /* It is a single table tablespace and the .ibd file is
  59                 missing: do nothing */
  60
  61 +               /* the log records should be treated here same reason
  62 +               for http://bugs.mysql.com/bug.php?id=43948 */
  63 +
  64 +               if (recv_recovery_is_on()) {
  65 +                       recv_addr_t*    recv_addr;
  66 +
  67 +                       mutex_enter(&(recv_sys->mutex));
  68 +
  69 +                       if (recv_sys->apply_log_recs == FALSE) {
  70 +                               mutex_exit(&(recv_sys->mutex));
  71 +                               goto not_to_recover;
  72 +                       }
  73 +
  74 +                       for (i = 0; i < n_stored; i++) {
  75 +                               /* recv_get_fil_addr_struct() */
  76 +                               recv_addr = HASH_GET_FIRST(recv_sys->addr_hash,
  77 +                                               hash_calc_hash(ut_fold_ulint_pair(space, page_nos[i]),
  78 +                                                       recv_sys->addr_hash));
  79 +                               while (recv_addr) {
  80 +                                       if ((recv_addr->space == space)
  81 +                                               && (recv_addr->page_no == page_nos[i])) {
  82 +                                               break;
  83 +                                       }
  84 +                                       recv_addr = HASH_GET_NEXT(addr_hash, recv_addr);
  85 +                               }
  86 +
  87 +                               if ((recv_addr == NULL)
  88 +                                   || (recv_addr->state == RECV_BEING_PROCESSED)
  89 +                                   || (recv_addr->state == RECV_PROCESSED)) {
  90 +                                       continue;
  91 +                               }
  92 +
  93 +                               recv_addr->state = RECV_PROCESSED;
  94 +
  95 +                               ut_a(recv_sys->n_addrs);
  96 +                               recv_sys->n_addrs--;
  97 +                       }
  98 +
  99 +                       mutex_exit(&(recv_sys->mutex));
 100 +
 101 +                       fprintf(stderr, " (cannot find space: %lu)", space);
 102 +               }
 103 +not_to_recover:
 104 +
 105                 return;
 106         }
 107
 108 --- a/storage/innobase/handler/ha_innodb.cc
 109 +++ b/storage/innobase/handler/ha_innodb.cc
 110 @@ -182,6 +182,7 @@
 111  #endif /* UNIV_LOG_ARCHIVE */
 112  static my_bool innobase_use_doublewrite                = TRUE;
 113  static my_bool innobase_use_checksums                  = TRUE;
 114 +static my_bool innobase_recovery_stats                 = TRUE;
 115  static my_bool innobase_locks_unsafe_for_binlog        = FALSE;
 116  static my_bool innobase_overwrite_relay_log_info       = FALSE;
 117  static my_bool innobase_rollback_on_timeout            = FALSE;
 118 @@ -2610,6 +2611,8 @@
 119
 120         srv_force_recovery = (ulint) innobase_force_recovery;
 121
 122 +       srv_recovery_stats = (ibool) innobase_recovery_stats;
 123 +
 124         srv_use_doublewrite_buf = (ibool) innobase_use_doublewrite;
 125         srv_use_checksums = (ibool) innobase_use_checksums;
 126
 127 @@ -11382,6 +11385,11 @@
 128    "The common part for InnoDB table spaces.",
 129    NULL, NULL, NULL);
 130
 131 +static MYSQL_SYSVAR_BOOL(recovery_stats, innobase_recovery_stats,
 132 +  PLUGIN_VAR_NOCMDARG | PLUGIN_VAR_READONLY,
 133 +  "Output statistics of recovery process after it.",
 134 +  NULL, NULL, FALSE);
 135 +
 136  static MYSQL_SYSVAR_BOOL(recovery_update_relay_log, innobase_overwrite_relay_log_info,
 137    PLUGIN_VAR_NOCMDARG | PLUGIN_VAR_READONLY,
 138    "During InnoDB crash recovery on slave overwrite relay-log.info "
 139 @@ -11870,6 +11878,7 @@
 140    MYSQL_SYSVAR(data_file_path),
 141    MYSQL_SYSVAR(data_home_dir),
 142    MYSQL_SYSVAR(doublewrite),
 143 +  MYSQL_SYSVAR(recovery_stats),
 144    MYSQL_SYSVAR(fast_shutdown),
 145    MYSQL_SYSVAR(file_io_threads),
 146    MYSQL_SYSVAR(read_io_threads),
 147 --- a/storage/innobase/include/log0recv.h
 148 +++ b/storage/innobase/include/log0recv.h
 149 @@ -438,6 +438,39 @@
 150         hash_table_t*   addr_hash;/*!< hash table of file addresses of pages */
 151         ulint           n_addrs;/*!< number of not processed hashed file
 152                                 addresses in the hash table */
 153 +
 154 +/* If you modified the following defines at original file,
 155 +   You should also modify them. */
 156 +/* defined in os0file.c */
 157 +#define OS_AIO_MERGE_N_CONSECUTIVE     64
 158 +/* defined in log0recv.c */
 159 +#define RECV_READ_AHEAD_AREA   32
 160 +       time_t          stats_recv_start_time;
 161 +       ulint           stats_recv_turns;
 162 +
 163 +       ulint           stats_read_requested_pages;
 164 +       ulint           stats_read_in_area[RECV_READ_AHEAD_AREA];
 165 +
 166 +       ulint           stats_read_io_pages;
 167 +       ulint           stats_read_io_consecutive[OS_AIO_MERGE_N_CONSECUTIVE];
 168 +       ulint           stats_write_io_pages;
 169 +       ulint           stats_write_io_consecutive[OS_AIO_MERGE_N_CONSECUTIVE];
 170 +
 171 +       ulint           stats_doublewrite_check_pages;
 172 +       ulint           stats_doublewrite_overwrite_pages;
 173 +
 174 +       ulint           stats_recover_pages_with_read;
 175 +       ulint           stats_recover_pages_without_read;
 176 +
 177 +       ulint           stats_log_recs;
 178 +       ulint           stats_log_len_sum;
 179 +
 180 +       ulint           stats_applied_log_recs;
 181 +       ulint           stats_applied_log_len_sum;
 182 +       ulint           stats_pages_already_new;
 183 +
 184 +       ib_uint64_t     stats_oldest_modified_lsn;
 185 +       ib_uint64_t     stats_newest_modified_lsn;
 186  };
 187
 188  /** The recovery system */
 189 --- a/storage/innobase/include/srv0srv.h
 190 +++ b/storage/innobase/include/srv0srv.h
 191 @@ -126,6 +126,8 @@
 192  extern ulint*  srv_data_file_sizes;
 193  extern ulint*  srv_data_file_is_raw_partition;
 194
 195 +extern ibool   srv_recovery_stats;
 196 +
 197  extern ibool   srv_auto_extend_last_data_file;
 198  extern ulint   srv_last_file_size_max;
 199  extern char**  srv_log_group_home_dirs;
 200 --- a/storage/innobase/log/log0recv.c
 201 +++ b/storage/innobase/log/log0recv.c
 202 @@ -187,6 +187,9 @@
 203
 204         recv_sys->heap = NULL;
 205         recv_sys->addr_hash = NULL;
 206 +
 207 +       recv_sys->stats_recv_start_time = time(NULL);
 208 +       recv_sys->stats_oldest_modified_lsn = IB_ULONGLONG_MAX;
 209  }
 210
 211  /********************************************************//**
 212 @@ -327,6 +330,11 @@
 213                 recv_n_pool_free_frames = 512;
 214         }
 215
 216 +       if (buf_pool_get_curr_size() >= (32 * 1024 * 1024)) {
 217 +               /* Buffer pool of size greater than 32 MB. */
 218 +               recv_n_pool_free_frames = 1024;
 219 +       }
 220 +
 221         recv_sys->buf = ut_malloc(RECV_PARSING_BUF_SIZE);
 222         recv_sys->len = 0;
 223         recv_sys->recovered_offset = 0;
 224 @@ -1363,6 +1371,11 @@
 225
 226         len = rec_end - body;
 227
 228 +       if (srv_recovery_stats) {
 229 +               recv_sys->stats_log_recs++;
 230 +               recv_sys->stats_log_len_sum += len;
 231 +       }
 232 +
 233         recv = mem_heap_alloc(recv_sys->heap, sizeof(recv_t));
 234         recv->type = type;
 235         recv->len = rec_end - body;
 236 @@ -1474,6 +1487,7 @@
 237         ib_uint64_t     start_lsn;
 238         ib_uint64_t     end_lsn;
 239         ib_uint64_t     page_lsn;
 240 +       ib_uint64_t     page_lsn_orig;
 241         ib_uint64_t     page_newest_lsn;
 242         ibool           modification_to_page;
 243  #ifndef UNIV_HOTBACKUP
 244 @@ -1496,6 +1510,8 @@
 245                                              buf_block_get_page_no(block));
 246
 247         if ((recv_addr == NULL)
 248 +               /* bugfix: http://bugs.mysql.com/bug.php?id=44140 */
 249 +           || (recv_addr->state == RECV_BEING_READ && !just_read_in)
 250             || (recv_addr->state == RECV_BEING_PROCESSED)
 251             || (recv_addr->state == RECV_PROCESSED)) {
 252
 253 @@ -1511,6 +1527,14 @@
 254
 255         recv_addr->state = RECV_BEING_PROCESSED;
 256
 257 +       if (srv_recovery_stats) {
 258 +               if (just_read_in) {
 259 +                       recv_sys->stats_recover_pages_with_read++;
 260 +               } else {
 261 +                       recv_sys->stats_recover_pages_without_read++;
 262 +               }
 263 +       }
 264 +
 265         mutex_exit(&(recv_sys->mutex));
 266
 267         mtr_start(&mtr);
 268 @@ -1540,6 +1564,7 @@
 269
 270         /* Read the newest modification lsn from the page */
 271         page_lsn = mach_read_from_8(page + FIL_PAGE_LSN);
 272 +       page_lsn_orig = page_lsn;
 273
 274  #ifndef UNIV_HOTBACKUP
 275         /* It may be that the page has been modified in the buffer
 276 @@ -1559,6 +1584,21 @@
 277         modification_to_page = FALSE;
 278         start_lsn = end_lsn = 0;
 279
 280 +       if (srv_recovery_stats) {
 281 +               mutex_enter(&(recv_sys->mutex));
 282 +               if (page_lsn_orig && recv_sys->stats_oldest_modified_lsn > page_lsn_orig) {
 283 +                       recv_sys->stats_oldest_modified_lsn = page_lsn_orig;
 284 +               }
 285 +               if (page_lsn_orig && recv_sys->stats_newest_modified_lsn < page_lsn_orig) {
 286 +                       recv_sys->stats_newest_modified_lsn = page_lsn_orig;
 287 +               }
 288 +               if (UT_LIST_GET_LAST(recv_addr->rec_list)->start_lsn
 289 +                   < page_lsn_orig) {
 290 +                       recv_sys->stats_pages_already_new++;
 291 +               }
 292 +               mutex_exit(&(recv_sys->mutex));
 293 +       }
 294 +
 295         recv = UT_LIST_GET_FIRST(recv_addr->rec_list);
 296
 297         while (recv) {
 298 @@ -1613,6 +1653,13 @@
 299                                                          buf + recv->len,
 300                                                          block, &mtr);
 301
 302 +                       if (srv_recovery_stats) {
 303 +                               mutex_enter(&(recv_sys->mutex));
 304 +                               recv_sys->stats_applied_log_recs++;
 305 +                               recv_sys->stats_applied_log_len_sum += recv->len;
 306 +                               mutex_exit(&(recv_sys->mutex));
 307 +                       }
 308 +
 309                         end_lsn = recv->start_lsn + recv->len;
 310                         mach_write_to_8(FIL_PAGE_LSN + page, end_lsn);
 311                         mach_write_to_8(UNIV_PAGE_SIZE
 312 @@ -1715,6 +1762,13 @@
 313                 }
 314         }
 315
 316 +       if (srv_recovery_stats && n) {
 317 +               mutex_enter(&(recv_sys->mutex));
 318 +               recv_sys->stats_read_requested_pages += n;
 319 +               recv_sys->stats_read_in_area[n - 1]++;
 320 +               mutex_exit(&(recv_sys->mutex));
 321 +       }
 322 +
 323         buf_read_recv_pages(FALSE, space, zip_size, page_nos, n);
 324         /*
 325         fprintf(stderr, "Recv pages at %lu n %lu\n", page_nos[0], n);
 326 @@ -1867,6 +1921,10 @@
 327
 328         if (has_printed) {
 329                 fprintf(stderr, "InnoDB: Apply batch completed\n");
 330 +
 331 +               if (srv_recovery_stats) {
 332 +                       recv_sys->stats_recv_turns++;
 333 +               }
 334         }
 335
 336         mutex_exit(&(recv_sys->mutex));
 337 @@ -3270,6 +3328,90 @@
 338         }
 339  #endif /* UNIV_DEBUG */
 340
 341 +       if (recv_needed_recovery && srv_recovery_stats) {
 342 +               ulint   flush_list_len = 0;
 343 +               ulint   i;
 344 +
 345 +               fprintf(stderr,
 346 +                       "InnoDB: Log records have been applied. The statistics that were gathered follow.\n");
 347 +
 348 +               fprintf(stderr,
 349 +                       "============================================================\n"
 350 +                       "-------------------\n"
 351 +                       "RECOVERY STATISTICS\n"
 352 +                       "-------------------\n");
 353 +               fprintf(stderr,
 354 +                       "Recovery time: %g sec. (%lu turns)\n",
 355 +                       difftime(time(NULL), recv_sys->stats_recv_start_time),
 356 +                       recv_sys->stats_recv_turns);
 357 +
 358 +               for (i = 0; i < srv_buf_pool_instances; i++) {
 359 +                       buf_pool_t*     buf_pool;
 360 +
 361 +                       buf_pool = buf_pool_from_array(i);
 362 +                       flush_list_len += UT_LIST_GET_LEN(buf_pool->flush_list);
 363 +               }
 364 +               fprintf(stderr,
 365 +                       "\n"
 366 +                       "Data page IO statistics\n"
 367 +                       "  Requested pages: %lu\n"
 368 +                       "  Read pages:      %lu\n"
 369 +                       "  Written pages:   %lu\n"
 370 +                       "  (Dirty blocks):  %lu\n",
 371 +                       recv_sys->stats_read_requested_pages,
 372 +                       recv_sys->stats_read_io_pages,
 373 +                       recv_sys->stats_write_io_pages,
 374 +                       flush_list_len);
 375 +
 376 +               fprintf(stderr,
 377 +                       "  Grouping IO [times]:\n"
 378 +                       "\tnumber of pages,\n"
 379 +                       "\t\tread request neighbors (in %d pages chunk),\n"
 380 +                       "\t\t\tcombined read IO,\n"
 381 +                       "\t\t\t\tcombined write IO\n",
 382 +                       RECV_READ_AHEAD_AREA);
 383 +               for (i = 0; i < ut_max(RECV_READ_AHEAD_AREA,
 384 +                                       OS_AIO_MERGE_N_CONSECUTIVE); i++) {
 385 +                       fprintf(stderr,
 386 +                               "\t%3lu,\t%lu,\t%lu,\t%lu\n", i + 1,
 387 +                               (i < RECV_READ_AHEAD_AREA) ?
 388 +                                       recv_sys->stats_read_in_area[i] : 0,
 389 +                               (i < OS_AIO_MERGE_N_CONSECUTIVE) ?
 390 +                                       recv_sys->stats_read_io_consecutive[i] : 0,
 391 +                               (i < OS_AIO_MERGE_N_CONSECUTIVE) ?
 392 +                                       recv_sys->stats_write_io_consecutive[i] : 0);
 393 +               }
 394 +
 395 +               fprintf(stderr,
 396 +                       "\n"
 397 +                       "Recovery process statistics\n"
 398 +                       "  Checked pages by doublewrite buffer: %lu\n"
 399 +                       "  Overwritten pages from doublewrite:  %lu\n"
 400 +                       "  Recovered pages by io_thread:        %lu\n"
 401 +                       "  Recovered pages by main thread:      %lu\n"
 402 +                       "  Parsed log records to apply:         %lu\n"
 403 +                       "            Sum of the length:         %lu\n"
 404 +                       "  Applied log records:                 %lu\n"
 405 +                       "            Sum of the length:         %lu\n"
 406 +                       "  Pages which are already new enough:  %lu (It may not be accurate, if turns > 1)\n"
 407 +                       "  Oldest page's LSN:                   %llu\n"
 408 +                       "  Newest page's LSN:                   %llu\n",
 409 +                       recv_sys->stats_doublewrite_check_pages,
 410 +                       recv_sys->stats_doublewrite_overwrite_pages,
 411 +                       recv_sys->stats_recover_pages_with_read,
 412 +                       recv_sys->stats_recover_pages_without_read,
 413 +                       recv_sys->stats_log_recs,
 414 +                       recv_sys->stats_log_len_sum,
 415 +                       recv_sys->stats_applied_log_recs,
 416 +                       recv_sys->stats_applied_log_len_sum,
 417 +                       recv_sys->stats_pages_already_new,
 418 +                       recv_sys->stats_oldest_modified_lsn,
 419 +                       recv_sys->stats_newest_modified_lsn);
 420 +
 421 +               fprintf(stderr,
 422 +                       "============================================================\n");
 423 +       }
 424 +
 425         if (recv_needed_recovery) {
 426                 trx_sys_print_mysql_master_log_pos();
 427                 trx_sys_print_mysql_binlog_offset();
 428 --- a/storage/innobase/os/os0file.c
 429 +++ b/storage/innobase/os/os0file.c
 430 @@ -43,6 +43,7 @@
 431  #include "srv0start.h"
 432  #include "fil0fil.h"
 433  #include "buf0buf.h"
 434 +#include "log0recv.h"
 435  #ifndef UNIV_HOTBACKUP
 436  # include "os0sync.h"
 437  # include "os0thread.h"
 438 @@ -4278,6 +4279,18 @@
 439                                            INFINITE);
 440         }
 441
 442 +       if (srv_recovery_stats && recv_recovery_is_on() && n_consecutive) {
 443 +               mutex_enter(&(recv_sys->mutex));
 444 +               if (slot->type == OS_FILE_READ) {
 445 +                       recv_sys->stats_read_io_pages += n_consecutive;
 446 +                       recv_sys->stats_read_io_consecutive[n_consecutive - 1]++;
 447 +               } else if (slot->type == OS_FILE_WRITE) {
 448 +                       recv_sys->stats_write_io_pages += n_consecutive;
 449 +                       recv_sys->stats_write_io_consecutive[n_consecutive - 1]++;
 450 +               }
 451 +               mutex_exit(&(recv_sys->mutex));
 452 +       }
 453 +
 454         os_mutex_enter(array->mutex);
 455
 456         if (srv_shutdown_state == SRV_SHUTDOWN_EXIT_THREADS
 457 --- a/storage/innobase/srv/srv0srv.c
 458 +++ b/storage/innobase/srv/srv0srv.c
 459 @@ -160,6 +160,8 @@
 460  /* size in database pages */
 461  UNIV_INTERN ulint*     srv_data_file_sizes = NULL;
 462
 463 +UNIV_INTERN ibool      srv_recovery_stats = FALSE;
 464 +
 465  /* if TRUE, then we auto-extend the last data file */
 466  UNIV_INTERN ibool      srv_auto_extend_last_data_file  = FALSE;
 467  /* if != 0, this tells the max size auto-extending may increase the
 468 --- a/storage/innobase/trx/trx0sys.c
 469 +++ b/storage/innobase/trx/trx0sys.c
 470 @@ -567,6 +567,12 @@
 471                                zip_size ? zip_size : UNIV_PAGE_SIZE,
 472                                read_buf, NULL);
 473
 474 +                       if (srv_recovery_stats && recv_recovery_is_on()) {
 475 +                               mutex_enter(&(recv_sys->mutex));
 476 +                               recv_sys->stats_doublewrite_check_pages++;
 477 +                               mutex_exit(&(recv_sys->mutex));
 478 +                       }
 479 +
 480                         /* Check if the page is corrupt */
 481
 482                         if (UNIV_UNLIKELY
 483 @@ -614,6 +620,13 @@
 484                                        zip_size, page_no, 0,
 485                                        zip_size ? zip_size : UNIV_PAGE_SIZE,
 486                                        page, NULL);
 487 +
 488 +                               if (srv_recovery_stats && recv_recovery_is_on()) {
 489 +                                       mutex_enter(&(recv_sys->mutex));
 490 +                                       recv_sys->stats_doublewrite_overwrite_pages++;
 491 +                                       mutex_exit(&(recv_sys->mutex));
 492 +                               }
 493 +
 494                                 fprintf(stderr,
 495                                         "InnoDB: Recovered the page from"
 496                                         " the doublewrite buffer.\n");