1 # name : innodb_dict_size_limit.patch
2 # introduced : 11 or before
3 # maintainer : Yasufumi
6 # Any small change to this file in the main branch
7 # should be done or reviewed by the maintainer!
8 diff -ruN a/storage/innobase/btr/btr0sea.c b/storage/innobase/btr/btr0sea.c
9 --- a/storage/innobase/btr/btr0sea.c 2010-11-03 07:01:13.000000000 +0900
10 +++ b/storage/innobase/btr/btr0sea.c 2010-12-03 15:45:47.503988924 +0900
11 @@ -1185,6 +1185,179 @@
15 +/************************************************************************
16 +Drops a page hash index based on index */
19 +btr_search_drop_page_hash_index_on_index(
20 +/*=====================================*/
21 + dict_index_t* index) /* in: record descriptor */
24 + hash_table_t* table;
32 + index_id_t index_id;
37 + mem_heap_t* heap = NULL;
39 + ibool released_search_latch;
41 + rw_lock_s_lock(&btr_search_latch);
43 + table = btr_search_sys->hash_index;
45 + for (j = 0; j < srv_buf_pool_instances; j++) {
46 + buf_pool_t* buf_pool;
48 + buf_pool = buf_pool_from_array(j);
51 + buf_chunk_t* chunks = buf_pool->chunks;
52 + buf_chunk_t* chunk = chunks + buf_pool->n_chunks;
54 + released_search_latch = FALSE;
56 + while (--chunk >= chunks) {
57 + block = chunk->blocks;
61 + for (; i--; block++) {
62 + if (buf_block_get_state(block)
63 + != BUF_BLOCK_FILE_PAGE
64 + || block->index != index
65 + || !block->is_hashed) {
69 + page = block->frame;
71 + /* from btr_search_drop_page_hash_index() */
72 + n_fields = block->curr_n_fields;
73 + n_bytes = block->curr_n_bytes;
76 + /* keeping latch order */
77 + rw_lock_s_unlock(&btr_search_latch);
78 + released_search_latch = TRUE;
79 + rw_lock_x_lock(&block->lock);
82 + ut_a(n_fields + n_bytes > 0);
84 + n_recs = page_get_n_recs(page);
86 + /* Calculate and cache fold values into an array for fast deletion
87 + from the hash index */
89 + folds = mem_alloc(n_recs * sizeof(ulint));
93 + rec = page_get_infimum_rec(page);
94 + rec = page_rec_get_next_low(rec, page_is_comp(page));
96 + index_id = btr_page_get_index_id(page);
98 + ut_a(index_id == index->id);
104 + while (!page_rec_is_supremum(rec)) {
105 + offsets = rec_get_offsets(rec, index, offsets,
106 + n_fields + (n_bytes > 0), &heap);
107 + ut_a(rec_offs_n_fields(offsets) == n_fields + (n_bytes > 0));
108 + fold = rec_fold(rec, offsets, n_fields, n_bytes, index_id);
110 + if (fold == prev_fold && prev_fold != 0) {
115 + /* Remove all hash nodes pointing to this page from the
118 + folds[n_cached] = fold;
121 + rec = page_rec_get_next_low(rec, page_rec_is_comp(rec));
125 + if (UNIV_LIKELY_NULL(heap)) {
126 + mem_heap_empty(heap);
129 + rw_lock_x_lock(&btr_search_latch);
131 + if (UNIV_UNLIKELY(!block->is_hashed)) {
135 + ut_a(block->index == index);
137 + if (UNIV_UNLIKELY(block->curr_n_fields != n_fields)
138 + || UNIV_UNLIKELY(block->curr_n_bytes != n_bytes)) {
139 + rw_lock_x_unlock(&btr_search_latch);
140 + rw_lock_x_unlock(&block->lock);
144 + rw_lock_s_lock(&btr_search_latch);
148 + for (i = 0; i < n_cached; i++) {
150 + ha_remove_all_nodes_to_page(table, folds[i], page);
153 + ut_a(index->search_info->ref_count > 0);
154 + index->search_info->ref_count--;
156 + block->is_hashed = FALSE;
157 + block->index = NULL;
160 +#if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG
161 + if (UNIV_UNLIKELY(block->n_pointers)) {
163 + ut_print_timestamp(stderr);
165 +"InnoDB: The adaptive hash index is corrupted. After dropping\n"
166 +"InnoDB: the hash index to a page of %s, %lu hash nodes still remain.\n",
167 + index->name, (ulong) block->n_pointers);
169 +#endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */
170 + rw_lock_x_unlock(&btr_search_latch);
171 + rw_lock_x_unlock(&block->lock);
175 + rw_lock_s_lock(&btr_search_latch);
178 + } while (released_search_latch);
181 + rw_lock_s_unlock(&btr_search_latch);
183 + if (UNIV_LIKELY_NULL(heap)) {
184 + mem_heap_free(heap);
188 /********************************************************************//**
189 Drops a page hash index when a page is freed from a fseg to the file system.
190 Drops possible hash index if the page happens to be in the buffer pool. */
191 diff -ruN a/storage/innobase/buf/buf0buf.c b/storage/innobase/buf/buf0buf.c
192 --- a/storage/innobase/buf/buf0buf.c 2011-02-01 18:00:03.000000000 +0900
193 +++ b/storage/innobase/buf/buf0buf.c 2011-02-01 18:01:59.000000000 +0900
194 @@ -294,14 +294,14 @@
195 # endif /* !PFS_SKIP_BUFFER_MUTEX_RWLOCK */
196 #endif /* UNIV_PFS_MUTEX || UNIV_PFS_RWLOCK */
198 -/** A chunk of buffers. The buffer pool is allocated in chunks. */
199 -struct buf_chunk_struct{
200 - ulint mem_size; /*!< allocated size of the chunk */
201 - ulint size; /*!< size of frames[] and blocks[] */
202 - void* mem; /*!< pointer to the memory area which
203 - was allocated for the frames */
204 - buf_block_t* blocks; /*!< array of buffer control blocks */
206 +/** A chunk of buffers. The buffer pool is allocated in chunks. (moved to buf0buf.h)*/
207 +//struct buf_chunk_struct{
208 +// ulint mem_size; /*!< allocated size of the chunk */
209 +// ulint size; /*!< size of frames[] and blocks[] */
210 +// void* mem; /*!< pointer to the memory area which
211 +// was allocated for the frames */
212 +// buf_block_t* blocks; /*!< array of buffer control blocks */
214 #endif /* !UNIV_HOTBACKUP */
216 /********************************************************************//**
217 diff -ruN a/storage/innobase/dict/dict0boot.c b/storage/innobase/dict/dict0boot.c
218 --- a/storage/innobase/dict/dict0boot.c 2010-11-03 07:01:13.000000000 +0900
219 +++ b/storage/innobase/dict/dict0boot.c 2010-12-03 15:45:47.503988924 +0900
222 /*-------------------------*/
223 table = dict_mem_table_create("SYS_TABLES", DICT_HDR_SPACE, 8, 0);
224 + table->n_mysql_handles_opened = 1; /* for pin */
226 dict_mem_table_add_col(table, heap, "NAME", DATA_BINARY, 0, 0);
227 dict_mem_table_add_col(table, heap, "ID", DATA_BINARY, 0, 0);
230 /*-------------------------*/
231 table = dict_mem_table_create("SYS_COLUMNS", DICT_HDR_SPACE, 7, 0);
232 + table->n_mysql_handles_opened = 1; /* for pin */
234 dict_mem_table_add_col(table, heap, "TABLE_ID", DATA_BINARY, 0, 0);
235 dict_mem_table_add_col(table, heap, "POS", DATA_INT, 0, 4);
238 /*-------------------------*/
239 table = dict_mem_table_create("SYS_INDEXES", DICT_HDR_SPACE, 7, 0);
240 + table->n_mysql_handles_opened = 1; /* for pin */
242 dict_mem_table_add_col(table, heap, "TABLE_ID", DATA_BINARY, 0, 0);
243 dict_mem_table_add_col(table, heap, "ID", DATA_BINARY, 0, 0);
246 /*-------------------------*/
247 table = dict_mem_table_create("SYS_FIELDS", DICT_HDR_SPACE, 3, 0);
248 + table->n_mysql_handles_opened = 1; /* for pin */
250 dict_mem_table_add_col(table, heap, "INDEX_ID", DATA_BINARY, 0, 0);
251 dict_mem_table_add_col(table, heap, "POS", DATA_INT, 0, 4);
252 diff -ruN a/storage/innobase/dict/dict0crea.c b/storage/innobase/dict/dict0crea.c
253 --- a/storage/innobase/dict/dict0crea.c 2010-11-03 07:01:13.000000000 +0900
254 +++ b/storage/innobase/dict/dict0crea.c 2010-12-03 15:45:47.521955810 +0900
255 @@ -1210,6 +1210,9 @@
256 /* Foreign constraint system tables have already been
257 created, and they are ok */
259 + table1->n_mysql_handles_opened = 1; /* for pin */
260 + table2->n_mysql_handles_opened = 1; /* for pin */
262 mutex_exit(&(dict_sys->mutex));
265 @@ -1291,6 +1294,11 @@
267 trx_commit_for_mysql(trx);
269 + table1 = dict_table_get_low("SYS_FOREIGN");
270 + table2 = dict_table_get_low("SYS_FOREIGN_COLS");
271 + table1->n_mysql_handles_opened = 1; /* for pin */
272 + table2->n_mysql_handles_opened = 1; /* for pin */
274 row_mysql_unlock_data_dictionary(trx);
276 trx_free_for_mysql(trx);
277 diff -ruN a/storage/innobase/dict/dict0dict.c b/storage/innobase/dict/dict0dict.c
278 --- a/storage/innobase/dict/dict0dict.c 2010-11-03 07:01:13.000000000 +0900
279 +++ b/storage/innobase/dict/dict0dict.c 2010-12-03 15:45:47.525953769 +0900
282 table = dict_table_get_on_id_low(table_id);
284 + dict_table_LRU_trim(table);
286 mutex_exit(&(dict_sys->mutex));
290 table->n_mysql_handles_opened++;
293 + dict_table_LRU_trim(table);
295 mutex_exit(&(dict_sys->mutex));
298 @@ -1260,6 +1264,64 @@
299 dict_mem_table_free(table);
302 +/**************************************************************************
303 +Frees tables from the end of table_LRU if the dictionary cache occupies
307 +dict_table_LRU_trim(
308 +/*================*/
309 + dict_table_t* self)
311 + dict_table_t* table;
312 + dict_table_t* prev_table;
313 + dict_foreign_t* foreign;
315 + ulint n_have_parent;
316 + ulint cached_foreign_tables;
318 +#ifdef UNIV_SYNC_DEBUG
319 + ut_ad(mutex_own(&(dict_sys->mutex)));
320 +#endif /* UNIV_SYNC_DEBUG */
323 + n_removed = n_have_parent = 0;
324 + table = UT_LIST_GET_LAST(dict_sys->table_LRU);
326 + while ( srv_dict_size_limit && table
327 + && ((dict_sys->table_hash->n_cells
328 + + dict_sys->table_id_hash->n_cells) * sizeof(hash_cell_t)
329 + + dict_sys->size) > srv_dict_size_limit ) {
330 + prev_table = UT_LIST_GET_PREV(table_LRU, table);
332 + if (table == self || table->n_mysql_handles_opened)
335 + cached_foreign_tables = 0;
336 + foreign = UT_LIST_GET_FIRST(table->foreign_list);
337 + while (foreign != NULL) {
338 + if (foreign->referenced_table)
339 + cached_foreign_tables++;
340 + foreign = UT_LIST_GET_NEXT(foreign_list, foreign);
343 + if (cached_foreign_tables == 0) {
344 + dict_table_remove_from_cache(table);
350 + table = prev_table;
353 + if ( srv_dict_size_limit && n_have_parent && n_removed
354 + && ((dict_sys->table_hash->n_cells
355 + + dict_sys->table_id_hash->n_cells) * sizeof(hash_cell_t)
356 + + dict_sys->size) > srv_dict_size_limit )
360 /****************************************************************//**
361 If the given column name is reserved for InnoDB system columns, return
363 @@ -1729,6 +1791,11 @@
364 ut_ad(index->magic_n == DICT_INDEX_MAGIC_N);
365 ut_ad(mutex_own(&(dict_sys->mutex)));
367 + /* remove all entry of the index from adaptive hash index,
368 + because removing from adaptive hash index needs dict_index */
369 + if (btr_search_enabled && srv_dict_size_limit)
370 + btr_search_drop_page_hash_index_on_index(index);
372 /* We always create search info whether or not adaptive
373 hash index is enabled or not. */
374 info = index->search_info;
375 diff -ruN a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc
376 --- a/storage/innobase/handler/ha_innodb.cc 2010-12-03 15:43:57.294986852 +0900
377 +++ b/storage/innobase/handler/ha_innodb.cc 2010-12-03 15:45:47.534959966 +0900
379 (char*) &export_vars.innodb_dblwr_pages_written, SHOW_LONG},
381 (char*) &export_vars.innodb_dblwr_writes, SHOW_LONG},
383 + (char*) &export_vars.innodb_dict_tables, SHOW_LONG},
384 {"have_atomic_builtins",
385 (char*) &export_vars.innodb_have_atomic_builtins, SHOW_BOOL},
387 @@ -11618,6 +11620,11 @@
388 "Choose method of innodb_adaptive_flushing. (native, [estimate], keep_average)",
389 NULL, innodb_adaptive_flushing_method_update, 1, &adaptive_flushing_method_typelib);
391 +static MYSQL_SYSVAR_ULONG(dict_size_limit, srv_dict_size_limit,
392 + PLUGIN_VAR_RQCMDARG,
393 + "Limit the allocated memory for dictionary cache. (0: unlimited)",
394 + NULL, NULL, 0, 0, LONG_MAX, 0);
396 static struct st_mysql_sys_var* innobase_system_variables[]= {
397 MYSQL_SYSVAR(additional_mem_pool_size),
398 MYSQL_SYSVAR(autoextend_increment),
399 @@ -11685,6 +11692,7 @@
400 MYSQL_SYSVAR(flush_neighbor_pages),
401 MYSQL_SYSVAR(read_ahead),
402 MYSQL_SYSVAR(adaptive_flushing_method),
403 + MYSQL_SYSVAR(dict_size_limit),
404 MYSQL_SYSVAR(use_sys_malloc),
405 MYSQL_SYSVAR(use_native_aio),
406 MYSQL_SYSVAR(change_buffering),
407 diff -ruN a/storage/innobase/ibuf/ibuf0ibuf.c b/storage/innobase/ibuf/ibuf0ibuf.c
408 --- a/storage/innobase/ibuf/ibuf0ibuf.c 2010-12-03 15:18:48.889024455 +0900
409 +++ b/storage/innobase/ibuf/ibuf0ibuf.c 2010-12-03 15:45:47.553025057 +0900
412 /* Use old-style record format for the insert buffer. */
413 table = dict_mem_table_create(IBUF_TABLE_NAME, IBUF_SPACE_ID, 1, 0);
414 + table->n_mysql_handles_opened = 1; /* for pin */
416 dict_mem_table_add_col(table, heap, "DUMMY_COLUMN", DATA_BINARY, 0, 0);
418 diff -ruN a/storage/innobase/include/btr0sea.h b/storage/innobase/include/btr0sea.h
419 --- a/storage/innobase/include/btr0sea.h 2010-11-03 07:01:13.000000000 +0900
420 +++ b/storage/innobase/include/btr0sea.h 2010-12-03 15:45:47.555024229 +0900
422 s- or x-latched, or an index page
423 for which we know that
424 block->buf_fix_count == 0 */
425 +/************************************************************************
426 +Drops a page hash index based on index */
429 +btr_search_drop_page_hash_index_on_index(
430 +/*=====================================*/
431 + dict_index_t* index); /* in: record descriptor */
432 /********************************************************************//**
433 Drops a page hash index when a page is freed from a fseg to the file system.
434 Drops possible hash index if the page happens to be in the buffer pool. */
435 diff -ruN a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h
436 --- a/storage/innobase/include/buf0buf.h 2011-02-01 18:00:03.000000000 +0900
437 +++ b/storage/innobase/include/buf0buf.h 2011-02-01 18:03:29.000000000 +0900
438 @@ -1553,6 +1553,15 @@
439 #define BUF_POOL_ZIP_FOLD_BPAGE(b) BUF_POOL_ZIP_FOLD((buf_block_t*) (b))
442 +/** A chunk of buffers. The buffer pool is allocated in chunks. */
443 +struct buf_chunk_struct{
444 + ulint mem_size; /*!< allocated size of the chunk */
445 + ulint size; /*!< size of frames[] and blocks[] */
446 + void* mem; /*!< pointer to the memory area which
447 + was allocated for the frames */
448 + buf_block_t* blocks; /*!< array of buffer control blocks */
451 /** @brief The buffer pool statistics structure. */
452 struct buf_pool_stat_struct{
453 ulint n_page_gets; /*!< number of page gets performed;
454 diff -ruN a/storage/innobase/include/dict0dict.h b/storage/innobase/include/dict0dict.h
455 --- a/storage/innobase/include/dict0dict.h 2010-11-03 07:01:13.000000000 +0900
456 +++ b/storage/innobase/include/dict0dict.h 2010-12-03 15:45:47.558024515 +0900
457 @@ -1170,6 +1170,12 @@
458 /*====================================*/
459 dict_table_t* table, /*!< in: table */
460 const char* name); /*!< in: name of the index to find */
464 +dict_table_LRU_trim(
465 +/*================*/
466 + dict_table_t* self);
467 /* Buffers for storing detailed information about the latest foreign key
468 and unique key errors */
469 extern FILE* dict_foreign_err_file;
470 diff -ruN a/storage/innobase/include/dict0dict.ic b/storage/innobase/include/dict0dict.ic
471 --- a/storage/innobase/include/dict0dict.ic 2010-11-03 07:01:13.000000000 +0900
472 +++ b/storage/innobase/include/dict0dict.ic 2010-12-03 15:45:47.560024398 +0900
474 HASH_SEARCH(name_hash, dict_sys->table_hash, table_fold,
475 dict_table_t*, table, ut_ad(table->cached),
476 !strcmp(table->name, table_name));
478 + /* make young in table_LRU */
480 + UT_LIST_REMOVE(table_LRU, dict_sys->table_LRU, table);
481 + UT_LIST_ADD_FIRST(table_LRU, dict_sys->table_LRU, table);
488 table = dict_load_table_on_id(table_id);
491 + /* make young in table_LRU */
493 + UT_LIST_REMOVE(table_LRU, dict_sys->table_LRU, table);
494 + UT_LIST_ADD_FIRST(table_LRU, dict_sys->table_LRU, table);
497 ut_ad(!table || table->cached);
499 /* TODO: should get the type information from MySQL */
500 diff -ruN a/storage/innobase/include/srv0srv.h b/storage/innobase/include/srv0srv.h
501 --- a/storage/innobase/include/srv0srv.h 2010-12-03 15:43:57.297067100 +0900
502 +++ b/storage/innobase/include/srv0srv.h 2010-12-03 15:45:47.562024404 +0900
504 extern ulint srv_read_ahead;
505 extern ulint srv_adaptive_flushing_method;
507 +extern ulint srv_dict_size_limit;
508 /*-------------------------------------------*/
510 extern ulint srv_n_rows_inserted;
512 ulint innodb_data_writes; /*!< I/O write requests */
513 ulint innodb_data_written; /*!< Data bytes written */
514 ulint innodb_data_reads; /*!< I/O read requests */
515 + ulint innodb_dict_tables;
516 ulint innodb_buffer_pool_pages_total; /*!< Buffer pool size */
517 ulint innodb_buffer_pool_pages_data; /*!< Data pages */
518 ulint innodb_buffer_pool_pages_dirty; /*!< Dirty data pages */
519 diff -ruN a/storage/innobase/srv/srv0srv.c b/storage/innobase/srv/srv0srv.c
520 --- a/storage/innobase/srv/srv0srv.c 2010-12-03 15:43:57.301024390 +0900
521 +++ b/storage/innobase/srv/srv0srv.c 2010-12-03 15:45:47.565023830 +0900
523 UNIV_INTERN ulint srv_enable_unsafe_group_commit = 0; /* 0:disable 1:enable */
524 UNIV_INTERN ulint srv_read_ahead = 3; /* 1: random 2: linear 3: Both */
525 UNIV_INTERN ulint srv_adaptive_flushing_method = 0; /* 0: native 1: estimate 2: keep_average */
527 +UNIV_INTERN ulint srv_dict_size_limit = 0;
528 /*-------------------------------------------*/
529 UNIV_INTERN ulong srv_n_spin_wait_rounds = 30;
530 UNIV_INTERN ulong srv_n_free_tickets_to_enter = 500;
531 @@ -2232,6 +2234,7 @@
532 export_vars.innodb_data_reads = os_n_file_reads;
533 export_vars.innodb_data_writes = os_n_file_writes;
534 export_vars.innodb_data_written = srv_data_written;
535 + export_vars.innodb_dict_tables= (dict_sys ? UT_LIST_GET_LEN(dict_sys->table_LRU) : 0);
536 export_vars.innodb_buffer_pool_read_requests = stat.n_page_gets;
537 export_vars.innodb_buffer_pool_write_requests
538 = srv_buf_pool_write_requests;