1 /*****************************************************************************
2
3 Copyright (c) 2012, 2021, Oracle and/or its affiliates.
4
5 This program is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License, version 2.0,
7 as published by the Free Software Foundation.
8
9 This program is also distributed with certain software (including
10 but not limited to OpenSSL) that is licensed under separate terms,
11 as designated in a particular file or component or in included license
12 documentation. The authors of MySQL hereby grant you an additional
13 permission to link the program and your derivative works with the
14 separately licensed software that they have included with MySQL.
15
16 This program is distributed in the hope that it will be useful,
17 but WITHOUT ANY WARRANTY; without even the implied warranty of
18 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 GNU General Public License, version 2.0, for more details.
20
21 You should have received a copy of the GNU General Public License along with
22 this program; if not, write to the Free Software Foundation, Inc.,
23 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA
24
25 *****************************************************************************/
26
27 /**************************************************//**
28 @file dict/dict0stats_bg.cc
29 Code used for background table and index stats gathering.
30
31 Created Apr 25, 2012 Vasil Dimov
32 *******************************************************/
33
34 #include "dict0dict.h"
35 #include "dict0stats.h"
36 #include "dict0stats_bg.h"
37 #include "row0mysql.h"
38 #include "srv0start.h"
39 #include "ut0new.h"
40 #include "fil0fil.h"
41
42 #ifdef UNIV_NONINL
43 # include "dict0stats_bg.ic"
44 #endif
45
46 #include <vector>
47
48 /** Minimum time interval between stats recalc for a given table */
49 #define MIN_RECALC_INTERVAL 10 /* seconds */
50
51 #define SHUTTING_DOWN() (srv_shutdown_state != SRV_SHUTDOWN_NONE)
52
53 /** Event to wake up the stats thread */
54 os_event_t dict_stats_event = NULL;
55
56 #ifdef UNIV_DEBUG
57 /** Used by SET GLOBAL innodb_dict_stats_disabled_debug = 1; */
58 my_bool innodb_dict_stats_disabled_debug;
59
60 static os_event_t dict_stats_disabled_event;
61 #endif /* UNIV_DEBUG */
62
63 /** This mutex protects the "recalc_pool" variable. */
64 static ib_mutex_t recalc_pool_mutex;
65
66 /** The number of tables that can be added to "recalc_pool" before
67 it is enlarged */
68 static const ulint RECALC_POOL_INITIAL_SLOTS = 128;
69
70 /** Allocator type, used by std::vector */
71 typedef ut_allocator<table_id_t>
72 recalc_pool_allocator_t;
73
74 /** The multitude of tables whose stats are to be automatically
75 recalculated - an STL vector */
76 typedef std::vector<table_id_t, recalc_pool_allocator_t>
77 recalc_pool_t;
78
79 /** Iterator type for iterating over the elements of objects of type
80 recalc_pool_t. */
81 typedef recalc_pool_t::iterator
82 recalc_pool_iterator_t;
83
84 /** Pool where we store information on which tables are to be processed
85 by background statistics gathering. */
86 static recalc_pool_t* recalc_pool;
87
88 /** Variable to initiate shutdown the dict stats thread. Note we don't
89 use 'srv_shutdown_state' because we want to shutdown dict stats thread
90 before purge thread. */
91 static bool dict_stats_start_shutdown;
92
93 /** Event to wait for shutdown of the dict stats thread */
94 static os_event_t dict_stats_shutdown_event;
95
96 /*****************************************************************//**
97 Initialize the recalc pool, called once during thread initialization. */
98 static
99 void
dict_stats_recalc_pool_init()100 dict_stats_recalc_pool_init()
101 /*=========================*/
102 {
103 ut_ad(!srv_read_only_mode);
104
105 const PSI_memory_key key = mem_key_dict_stats_bg_recalc_pool_t;
106
107 recalc_pool = UT_NEW(recalc_pool_t(recalc_pool_allocator_t(key)), key);
108
109 recalc_pool->reserve(RECALC_POOL_INITIAL_SLOTS);
110 }
111
112 /*****************************************************************//**
113 Free the resources occupied by the recalc pool, called once during
114 thread de-initialization. */
115 static
116 void
dict_stats_recalc_pool_deinit()117 dict_stats_recalc_pool_deinit()
118 /*===========================*/
119 {
120 ut_ad(!srv_read_only_mode);
121
122 recalc_pool->clear();
123
124 UT_DELETE(recalc_pool);
125 }
126
127 /*****************************************************************//**
128 Add a table to the recalc pool, which is processed by the
129 background stats gathering thread. Only the table id is added to the
130 list, so the table can be closed after being enqueued and it will be
131 opened when needed. If the table does not exist later (has been DROPped),
132 then it will be removed from the pool and skipped. */
133 void
dict_stats_recalc_pool_add(const dict_table_t * table)134 dict_stats_recalc_pool_add(
135 /*=======================*/
136 const dict_table_t* table) /*!< in: table to add */
137 {
138 ut_ad(!srv_read_only_mode);
139
140 mutex_enter(&recalc_pool_mutex);
141
142 /* quit if already in the list */
143 for (recalc_pool_iterator_t iter = recalc_pool->begin();
144 iter != recalc_pool->end();
145 ++iter) {
146
147 if (*iter == table->id) {
148 mutex_exit(&recalc_pool_mutex);
149 return;
150 }
151 }
152
153 recalc_pool->push_back(table->id);
154
155 mutex_exit(&recalc_pool_mutex);
156
157 os_event_set(dict_stats_event);
158 }
159
160 /*****************************************************************//**
161 Get a table from the auto recalc pool. The returned table id is removed
162 from the pool.
163 @return true if the pool was non-empty and "id" was set, false otherwise */
164 static
165 bool
dict_stats_recalc_pool_get(table_id_t * id)166 dict_stats_recalc_pool_get(
167 /*=======================*/
168 table_id_t* id) /*!< out: table id, or unmodified if list is
169 empty */
170 {
171 ut_ad(!srv_read_only_mode);
172
173 mutex_enter(&recalc_pool_mutex);
174
175 if (recalc_pool->empty()) {
176 mutex_exit(&recalc_pool_mutex);
177 return(false);
178 }
179
180 *id = recalc_pool->at(0);
181
182 recalc_pool->erase(recalc_pool->begin());
183
184 mutex_exit(&recalc_pool_mutex);
185
186 return(true);
187 }
188
189 /*****************************************************************//**
190 Delete a given table from the auto recalc pool.
191 dict_stats_recalc_pool_del() */
192 void
dict_stats_recalc_pool_del(const dict_table_t * table)193 dict_stats_recalc_pool_del(
194 /*=======================*/
195 const dict_table_t* table) /*!< in: table to remove */
196 {
197 ut_ad(!srv_read_only_mode);
198 ut_ad(mutex_own(&dict_sys->mutex));
199
200 mutex_enter(&recalc_pool_mutex);
201
202 ut_ad(table->id > 0);
203
204 for (recalc_pool_iterator_t iter = recalc_pool->begin();
205 iter != recalc_pool->end();
206 ++iter) {
207
208 if (*iter == table->id) {
209 /* erase() invalidates the iterator */
210 recalc_pool->erase(iter);
211 break;
212 }
213 }
214
215 mutex_exit(&recalc_pool_mutex);
216 }
217
218 /*****************************************************************//**
219 Wait until background stats thread has stopped using the specified table.
220 The caller must have locked the data dictionary using
221 row_mysql_lock_data_dictionary() and this function may unlock it temporarily
222 and restore the lock before it exits.
223 The background stats thread is guaranteed not to start using the specified
224 table after this function returns and before the caller unlocks the data
225 dictionary because it sets the BG_STAT_IN_PROGRESS bit in table->stats_bg_flag
226 under dict_sys->mutex. */
227 void
dict_stats_wait_bg_to_stop_using_table(dict_table_t * table,trx_t * trx)228 dict_stats_wait_bg_to_stop_using_table(
229 /*===================================*/
230 dict_table_t* table, /*!< in/out: table */
231 trx_t* trx) /*!< in/out: transaction to use for
232 unlocking/locking the data dict */
233 {
234 while (!dict_stats_stop_bg(table)) {
235 DICT_BG_YIELD(trx);
236 }
237 }
238
239 /*****************************************************************//**
240 Initialize global variables needed for the operation of dict_stats_thread()
241 Must be called before dict_stats_thread() is started. */
242 void
dict_stats_thread_init()243 dict_stats_thread_init()
244 /*====================*/
245 {
246 ut_a(!srv_read_only_mode);
247
248 dict_stats_event = os_event_create(0);
249 dict_stats_shutdown_event = os_event_create(0);
250
251 ut_d(dict_stats_disabled_event = os_event_create(0));
252
253 /* The recalc_pool_mutex is acquired from:
254 1) the background stats gathering thread before any other latch
255 and released without latching anything else in between (thus
256 any level would do here)
257 2) from row_update_statistics_if_needed()
258 and released without latching anything else in between. We know
259 that dict_sys->mutex (SYNC_DICT) is not acquired when
260 row_update_statistics_if_needed() is called and it may be acquired
261 inside that function (thus a level <=SYNC_DICT would do).
262 3) from row_drop_table_for_mysql() after dict_sys->mutex (SYNC_DICT)
263 and dict_operation_lock (SYNC_DICT_OPERATION) have been locked
264 (thus a level <SYNC_DICT && <SYNC_DICT_OPERATION would do)
265 So we choose SYNC_STATS_AUTO_RECALC to be about below SYNC_DICT. */
266
267 mutex_create(LATCH_ID_RECALC_POOL, &recalc_pool_mutex);
268
269 dict_stats_recalc_pool_init();
270 }
271
272 /*****************************************************************//**
273 Free resources allocated by dict_stats_thread_init(), must be called
274 after dict_stats_thread() has exited. */
275 void
dict_stats_thread_deinit()276 dict_stats_thread_deinit()
277 /*======================*/
278 {
279 ut_a(!srv_read_only_mode);
280 ut_ad(!srv_dict_stats_thread_active);
281
282 dict_stats_recalc_pool_deinit();
283
284 mutex_free(&recalc_pool_mutex);
285
286 #ifdef UNIV_DEBUG
287 os_event_destroy(dict_stats_disabled_event);
288 dict_stats_disabled_event = NULL;
289 #endif /* UNIV_DEBUG */
290
291 os_event_destroy(dict_stats_event);
292 os_event_destroy(dict_stats_shutdown_event);
293 dict_stats_event = NULL;
294 dict_stats_shutdown_event = NULL;
295 dict_stats_start_shutdown = false;
296 }
297
298 /*****************************************************************//**
299 Get the first table that has been added for auto recalc and eventually
300 update its stats. */
301 static
302 void
dict_stats_process_entry_from_recalc_pool()303 dict_stats_process_entry_from_recalc_pool()
304 /*=======================================*/
305 {
306 table_id_t table_id;
307
308 ut_ad(!srv_read_only_mode);
309
310 /* pop the first table from the auto recalc pool */
311 if (!dict_stats_recalc_pool_get(&table_id)) {
312 /* no tables for auto recalc */
313 return;
314 }
315
316 dict_table_t* table;
317
318 mutex_enter(&dict_sys->mutex);
319
320 table = dict_table_open_on_id(table_id, TRUE, DICT_TABLE_OP_NORMAL);
321
322 if (table == NULL) {
323 /* table does not exist, must have been DROPped
324 after its id was enqueued */
325 mutex_exit(&dict_sys->mutex);
326 return;
327 }
328
329 if (fil_space_is_being_truncated(table->space)) {
330 dict_table_close(table, TRUE, FALSE);
331 mutex_exit(&dict_sys->mutex);
332 return;
333 }
334
335 /* Check whether table is corrupted */
336 if (table->corrupted) {
337 dict_table_close(table, TRUE, FALSE);
338 mutex_exit(&dict_sys->mutex);
339 return;
340 }
341
342 table->stats_bg_flag = BG_STAT_IN_PROGRESS;
343
344 mutex_exit(&dict_sys->mutex);
345
346 /* ut_time_monotonic() could be expensive, the current function
347 is called once every time a table has been changed more than 10% and
348 on a system with lots of small tables, this could become hot. If we
349 find out that this is a problem, then the check below could eventually
350 be replaced with something else, though a time interval is the natural
351 approach. */
352
353 if (ut_time_monotonic() - table->stats_last_recalc < MIN_RECALC_INTERVAL) {
354
355 /* Stats were (re)calculated not long ago. To avoid
356 too frequent stats updates we put back the table on
357 the auto recalc list and do nothing. */
358
359 dict_stats_recalc_pool_add(table);
360
361 } else {
362
363 dict_stats_update(table, DICT_STATS_RECALC_PERSISTENT);
364 }
365
366 mutex_enter(&dict_sys->mutex);
367
368 table->stats_bg_flag = BG_STAT_NONE;
369
370 dict_table_close(table, TRUE, FALSE);
371
372 mutex_exit(&dict_sys->mutex);
373 }
374
375 #ifdef UNIV_DEBUG
376 /** Disables dict stats thread. It's used by:
377 SET GLOBAL innodb_dict_stats_disabled_debug = 1 (0).
378 @param[in] thd thread handle
379 @param[in] var pointer to system variable
380 @param[out] var_ptr where the formal string goes
381 @param[in] save immediate result from check function */
382 void
dict_stats_disabled_debug_update(THD * thd,struct st_mysql_sys_var * var,void * var_ptr,const void * save)383 dict_stats_disabled_debug_update(
384 THD* thd,
385 struct st_mysql_sys_var* var,
386 void* var_ptr,
387 const void* save)
388 {
389 /* This method is protected by mutex, as every SET GLOBAL .. */
390 ut_ad(dict_stats_disabled_event != NULL);
391
392 const bool disable = *static_cast<const my_bool*>(save);
393
394 const int64_t sig_count = os_event_reset(dict_stats_disabled_event);
395
396 innodb_dict_stats_disabled_debug = disable;
397
398 if (disable) {
399 os_event_set(dict_stats_event);
400 os_event_wait_low(dict_stats_disabled_event, sig_count);
401 }
402 }
403 #endif /* UNIV_DEBUG */
404
405 /*****************************************************************//**
406 This is the thread for background stats gathering. It pops tables, from
407 the auto recalc list and proceeds them, eventually recalculating their
408 statistics.
409 @return this function does not return, it calls os_thread_exit() */
410 extern "C"
411 os_thread_ret_t
DECLARE_THREAD(dict_stats_thread)412 DECLARE_THREAD(dict_stats_thread)(
413 /*==============================*/
414 void* arg MY_ATTRIBUTE((unused))) /*!< in: a dummy parameter
415 required by os_thread_create */
416 {
417 ut_a(!srv_read_only_mode);
418
419 my_thread_init();
420
421 #ifdef UNIV_PFS_THREAD
422 pfs_register_thread(dict_stats_thread_key);
423 #endif /* UNIV_PFS_THREAD */
424
425 srv_dict_stats_thread_active = TRUE;
426
427 while (!dict_stats_start_shutdown) {
428
429 /* Wake up periodically even if not signaled. This is
430 because we may lose an event - if the below call to
431 dict_stats_process_entry_from_recalc_pool() puts the entry back
432 in the list, the os_event_set() will be lost by the subsequent
433 os_event_reset(). */
434 os_event_wait_time(
435 dict_stats_event, MIN_RECALC_INTERVAL * 1000000);
436
437 #ifdef UNIV_DEBUG
438 while (innodb_dict_stats_disabled_debug) {
439 os_event_set(dict_stats_disabled_event);
440 if (dict_stats_start_shutdown) {
441 break;
442 }
443 os_event_wait_time(
444 dict_stats_event, 100000);
445 }
446 #endif /* UNIV_DEBUG */
447
448 if (dict_stats_start_shutdown) {
449 break;
450 }
451
452 dict_stats_process_entry_from_recalc_pool();
453
454 os_event_reset(dict_stats_event);
455 }
456
457 srv_dict_stats_thread_active = FALSE;
458
459 os_event_set(dict_stats_shutdown_event);
460 my_thread_end();
461
462 /* We count the number of threads in os_thread_exit(). A created
463 thread should always use that to exit instead of return(). */
464 os_thread_exit();
465
466 OS_THREAD_DUMMY_RETURN;
467 }
468
469 /** Shutdown the dict stats thread. */
470 void
dict_stats_shutdown()471 dict_stats_shutdown()
472 {
473 dict_stats_start_shutdown = true;
474 os_event_set(dict_stats_event);
475 os_event_wait(dict_stats_shutdown_event);
476 }
477