1 /*
2 ** Zabbix
3 ** Copyright (C) 2001-2021 Zabbix SIA
4 **
5 ** This program is free software; you can redistribute it and/or modify
6 ** it under the terms of the GNU General Public License as published by
7 ** the Free Software Foundation; either version 2 of the License, or
8 ** (at your option) any later version.
9 **
10 ** This program is distributed in the hope that it will be useful,
11 ** but WITHOUT ANY WARRANTY; without even the implied warranty of
12 ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 ** GNU General Public License for more details.
14 **
15 ** You should have received a copy of the GNU General Public License
16 ** along with this program; if not, write to the Free Software
17 ** Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 **/
19
20 #include "common.h"
21 #include "log.h"
22 #include "memalloc.h"
23 #include "ipc.h"
24 #include "dbcache.h"
25 #include "zbxhistory.h"
26
27 #include "valuecache.h"
28
29 #include "vectorimpl.h"
30
31 /*
32 * The cache (zbx_vc_cache_t) is organized as a hashset of item records (zbx_vc_item_t).
33 *
34 * Each record holds item data (itemid, value_type), statistics (hits, last access time,...)
35 * and the historical data (timestamp,value pairs in ascending order).
36 *
37 * The historical data are stored from largest request (+timeshift) range to the
38 * current time. The data is automatically fetched from DB whenever a request
39 * exceeds cached value range.
40 *
41 * In addition to active range value cache tracks item range for last 24 hours. Once
42 * per day the active range is updated with daily range and the daily range is reset.
43 *
44 * If an item is already being cached the new values are automatically added to the cache
45 * after being written into database.
46 *
47 * When cache runs out of memory to store new items it enters in low memory mode.
48 * In low memory mode cache continues to function as before with few restrictions:
49 * 1) items that weren't accessed during the last day are removed from cache.
50 * 2) items with worst hits/values ratio might be removed from cache to free the space.
51 * 3) no new items are added to the cache.
52 *
53 * The low memory mode can't be turned off - it will persist until server is rebooted.
54 * In low memory mode a warning message is written into log every 5 minutes.
55 */
56
57 /* the period of low memory warning messages */
58 #define ZBX_VC_LOW_MEMORY_WARNING_PERIOD (5 * SEC_PER_MIN)
59
60 /* time period after which value cache will switch back to normal mode */
61 #define ZBX_VC_LOW_MEMORY_RESET_PERIOD SEC_PER_DAY
62
63 #define ZBX_VC_LOW_MEMORY_ITEM_PRINT_LIMIT 25
64
65 static zbx_mem_info_t *vc_mem = NULL;
66
67 zbx_rwlock_t vc_lock = ZBX_RWLOCK_NULL;
68
69 /* value cache enable/disable flags */
70 #define ZBX_VC_DISABLED 0
71 #define ZBX_VC_ENABLED 1
72
73 /* value cache state, after initialization value cache is always disabled */
74 static int vc_state = ZBX_VC_DISABLED;
75
76 /* the value cache size */
77 extern zbx_uint64_t CONFIG_VALUE_CACHE_SIZE;
78
79 ZBX_MEM_FUNC_IMPL(__vc, vc_mem)
80
81 #define VC_STRPOOL_INIT_SIZE (1000)
82 #define VC_ITEMS_INIT_SIZE (1000)
83
84 #define VC_MAX_NANOSECONDS 999999999
85
86 #define VC_MIN_RANGE SEC_PER_MIN
87
88 /* the range synchronization period in hours */
89 #define ZBX_VC_RANGE_SYNC_PERIOD 24
90
91 #define ZBX_VC_ITEM_EXPIRE_PERIOD SEC_PER_DAY
92
93 /* the data chunk used to store data fragment */
94 typedef struct zbx_vc_chunk
95 {
96 /* a pointer to the previous chunk or NULL if this is the tail chunk */
97 struct zbx_vc_chunk *prev;
98
99 /* a pointer to the next chunk or NULL if this is the head chunk */
100 struct zbx_vc_chunk *next;
101
102 /* the index of first (oldest) value in chunk */
103 int first_value;
104
105 /* the index of last (newest) value in chunk */
106 int last_value;
107
108 /* the number of item value slots in chunk */
109 int slots_num;
110
111 /* the item value data */
112 zbx_history_record_t slots[1];
113 }
114 zbx_vc_chunk_t;
115
116 /* min/max number number of item history values to store in chunk */
117
118 #define ZBX_VC_MIN_CHUNK_RECORDS 2
119
120 /* the maximum number is calculated so that the chunk size does not exceed 64KB */
121 #define ZBX_VC_MAX_CHUNK_RECORDS ((64 * ZBX_KIBIBYTE - sizeof(zbx_vc_chunk_t)) / \
122 sizeof(zbx_history_record_t) + 1)
123
124 /* the value cache item data */
125 typedef struct
126 {
127 /* the item id */
128 zbx_uint64_t itemid;
129
130 /* the item value type */
131 unsigned char value_type;
132
133 /* the item status flags (ZBX_ITEM_STATUS_*) */
134 unsigned char status;
135
136 /* the hour when the current/global range sync was done */
137 unsigned char range_sync_hour;
138
139 /* The total number of item values in cache. */
140 /* Used to evaluate if the item must be dropped from cache */
141 /* in low memory situation. */
142 int values_total;
143
144 /* The last time when item cache was accessed. */
145 /* Used to evaluate if the item must be dropped from cache */
146 /* in low memory situation. */
147 int last_accessed;
148
149 /* The range of the largest request in seconds. */
150 /* Used to determine if data can be removed from cache. */
151 int active_range;
152
153 /* The range for last 24 hours since active_range update. */
154 /* Once per day the active_range is synchronized (updated) */
155 /* with daily_range and the daily range is reset. */
156 int daily_range;
157
158 /* The timestamp marking the oldest value that is guaranteed */
159 /* to be cached. */
160 /* The db_cached_from value is based on actual requests made */
161 /* to database and is used to check if the requested time */
162 /* interval should be cached. */
163 int db_cached_from;
164
165 /* The maximum number of values returned by one request */
166 /* during last hourly interval. */
167 int last_hourly_num;
168
169 /* The maximum number of values returned by one request */
170 /* during current hourly interval. */
171 int hourly_num;
172
173 /* The hour of hourly_num */
174 int hour;
175
176 /* The number of cache hits for this item. */
177 /* Used to evaluate if the item must be dropped from cache */
178 /* in low memory situation. */
179 zbx_uint64_t hits;
180
181 /* the last (newest) chunk of item history data */
182 zbx_vc_chunk_t *head;
183
184 /* the first (oldest) chunk of item history data */
185 zbx_vc_chunk_t *tail;
186 }
187 zbx_vc_item_t;
188
189 /* the value cache data */
190 typedef struct
191 {
192 /* the number of cache hits, used for statistics */
193 zbx_uint64_t hits;
194
195 /* the number of cache misses, used for statistics */
196 zbx_uint64_t misses;
197
198 /* value cache operating mode - see ZBX_VC_MODE_* defines */
199 int mode;
200
201 /* time when cache operating mode was changed */
202 int mode_time;
203
204 /* timestamp of the last low memory warning message */
205 int last_warning_time;
206
207 /* the minimum number of bytes to be freed when cache runs out of space */
208 size_t min_free_request;
209
210 /* the cached items */
211 zbx_hashset_t items;
212
213 /* the string pool for str, text and log item values */
214 zbx_hashset_t strpool;
215 }
216 zbx_vc_cache_t;
217
218 /* the item weight data, used to determine if item can be removed from cache */
219 typedef struct
220 {
221 /* a pointer to the value cache item */
222 zbx_vc_item_t *item;
223
224 /* the item 'weight' - <number of hits> / <number of cache records> */
225 double weight;
226 }
227 zbx_vc_item_weight_t;
228
229 ZBX_VECTOR_DECL(vc_itemweight, zbx_vc_item_weight_t)
230 ZBX_VECTOR_IMPL(vc_itemweight, zbx_vc_item_weight_t)
231
232 typedef enum
233 {
234 ZBX_VC_UPDATE_STATS,
235 ZBX_VC_UPDATE_RANGE
236 }
237 zbx_vc_item_update_type_t;
238
239 enum
240 {
241 ZBX_VC_UPDATE_STATS_HITS,
242 ZBX_VC_UPDATE_STATS_MISSES
243 };
244
245 enum
246 {
247 ZBX_VC_UPDATE_RANGE_SECONDS,
248 ZBX_VC_UPDATE_RANGE_NOW
249 };
250
251 typedef struct
252 {
253 zbx_uint64_t itemid;
254 zbx_vc_item_update_type_t type;
255 int data[2];
256 }
257 zbx_vc_item_update_t;
258
259 ZBX_VECTOR_DECL(vc_itemupdate, zbx_vc_item_update_t)
260 ZBX_VECTOR_IMPL(vc_itemupdate, zbx_vc_item_update_t)
261
262 static zbx_vector_vc_itemupdate_t vc_itemupdates;
263
vc_cache_item_update(zbx_uint64_t itemid,zbx_vc_item_update_type_t type,int arg1,int arg2)264 static void vc_cache_item_update(zbx_uint64_t itemid, zbx_vc_item_update_type_t type, int arg1, int arg2)
265 {
266 zbx_vc_item_update_t *update;
267
268 if (vc_itemupdates.values_num == vc_itemupdates.values_alloc)
269 zbx_vector_vc_itemupdate_reserve(&vc_itemupdates, vc_itemupdates.values_alloc * 1.5);
270
271 update = &vc_itemupdates.values[vc_itemupdates.values_num++];
272 update->itemid = itemid;
273 update->type = type;
274 update->data[0] = arg1;
275 update->data[1] = arg2;
276 }
277
278 /* the value cache */
279 static zbx_vc_cache_t *vc_cache = NULL;
280
281 #define RDLOCK_CACHE zbx_rwlock_rdlock(vc_lock);
282 #define WRLOCK_CACHE zbx_rwlock_wrlock(vc_lock);
283 #define UNLOCK_CACHE zbx_rwlock_unlock(vc_lock);
284
285 /* function prototypes */
286 static void vc_history_record_copy(zbx_history_record_t *dst, const zbx_history_record_t *src, int value_type);
287 static void vc_history_record_vector_clean(zbx_vector_history_record_t *vector, int value_type);
288
289 static size_t vch_item_free_cache(zbx_vc_item_t *item);
290 static size_t vch_item_free_chunk(zbx_vc_item_t *item, zbx_vc_chunk_t *chunk);
291 static int vch_item_add_values_at_tail(zbx_vc_item_t *item, const zbx_history_record_t *values, int values_num);
292 static void vch_item_clean_cache(zbx_vc_item_t *item);
293
294 /*********************************************************************************
295 * *
296 * Function: vc_db_read_values_by_time *
297 * *
298 * Purpose: reads item history data from database *
299 * *
300 * Parameters: itemid - [IN] the itemid *
301 * value_type - [IN] the value type (see ITEM_VALUE_TYPE_* defs) *
302 * values - [OUT] the item history data values *
303 * range_start - [IN] the interval start time *
304 * range_end - [IN] the interval end time *
305 * *
306 * Return value: SUCCEED - the history data were read successfully *
307 * FAIL - otherwise *
308 * *
309 * Comments: This function reads all values from the specified range *
310 * [range_start, range_end] *
311 * *
312 *********************************************************************************/
vc_db_read_values_by_time(zbx_uint64_t itemid,int value_type,zbx_vector_history_record_t * values,int range_start,int range_end)313 static int vc_db_read_values_by_time(zbx_uint64_t itemid, int value_type, zbx_vector_history_record_t *values,
314 int range_start, int range_end)
315 {
316 /* decrement interval start point because interval starting point is excluded by history backend */
317 if (0 != range_start)
318 range_start--;
319
320 return zbx_history_get_values(itemid, value_type, range_start, 0, range_end, values);
321 }
322
323 /************************************************************************************
324 * *
325 * Function: vc_db_read_values_by_time_and_count *
326 * *
327 * Purpose: reads item history data from database *
328 * *
329 * Parameters: itemid - [IN] the itemid *
330 * value_type - [IN] the value type (see ITEM_VALUE_TYPE_* defs) *
331 * values - [OUT] the item history data values *
332 * range_start - [IN] the interval start time *
333 * count - [IN] the number of values to read *
334 * range_end - [IN] the interval end time *
335 * ts - [IN] the requested timestamp *
336 * *
337 * Return value: SUCCEED - the history data were read successfully *
338 * FAIL - otherwise *
339 * *
340 * Comments: This function will return the smallest data interval on seconds scale *
341 * that includes the requested values. *
342 * *
343 ************************************************************************************/
vc_db_read_values_by_time_and_count(zbx_uint64_t itemid,int value_type,zbx_vector_history_record_t * values,int range_start,int count,int range_end,const zbx_timespec_t * ts)344 static int vc_db_read_values_by_time_and_count(zbx_uint64_t itemid, int value_type,
345 zbx_vector_history_record_t *values, int range_start, int count, int range_end,
346 const zbx_timespec_t *ts)
347 {
348 int first_timestamp, last_timestamp, i, left = 0, values_start;
349
350 /* remember the number of values already stored in values vector */
351 values_start = values->values_num;
352
353 if (0 != range_start)
354 range_start--;
355
356 /* Count based requests can 'split' the data of oldest second. For example if we have */
357 /* values with timestamps Ta.0 Tb.0, Tb.5, Tc.0 then requesting 2 values from [0, Tc] */
358 /* range will return Tb.5, Tc.0,leaving Tb.0 value in database. However because */
359 /* second is the smallest time unit history backends can work with, data must be cached */
360 /* by second intervals - it cannot have some values from Tb cached and some not. */
361 /* This is achieved by two means: */
362 /* 1) request more (by one) values than we need. In most cases there will be no */
363 /* multiple values per second (exceptions are logs and trapper items) - for example */
364 /* Ta.0, Tb.0, Tc.0. We need 2 values from Tc. Requesting 3 values gets us */
365 /* Ta.0, Tb.0, Tc.0. As Ta != Tb we can be sure that all values from the last */
366 /* timestamp (Tb) have been cached. So we can drop Ta.0 and return Tb.0, Tc.0. */
367 /* 2) Re-read the last second. For example if we have values with timestamps */
368 /* Ta.0 Tb.0, Tb.5, Tc.0, then requesting 3 values from Tc gets us Tb.0, Tb.5, Tc.0.*/
369 /* Now we cannot be sure that there are no more values with Tb.* timestamp. So the */
370 /* only thing we can do is to: */
371 /* a) remove values with Tb.* timestamp from result, */
372 /* b) read all values with Tb.* timestamp from database, */
373 /* c) add read values to the result. */
374 if (FAIL == zbx_history_get_values(itemid, value_type, range_start, count + 1, range_end, values))
375 return FAIL;
376
377 /* returned less than requested - all values are read */
378 if (count > values->values_num - values_start)
379 return SUCCEED;
380
381 /* Check if some of values aren't past the required range. For example we have values */
382 /* with timestamps Ta.0, Tb.0, Tb.5. As history backends work on seconds interval we */
383 /* can only request values from Tb which would include Tb.5, even if the requested */
384 /* end timestamp was less (for example Tb.0); */
385
386 /* values from history backend are sorted in descending order by timestamp seconds */
387 first_timestamp = values->values[values->values_num - 1].timestamp.sec;
388 last_timestamp = values->values[values_start].timestamp.sec;
389
390 for (i = values_start; i < values->values_num && values->values[i].timestamp.sec == last_timestamp; i++)
391 {
392 if (0 > zbx_timespec_compare(ts, &values->values[i].timestamp))
393 left++;
394 }
395
396 /* read missing data */
397 if (0 != left)
398 {
399 int offset;
400
401 /* drop the first (oldest) second to ensure range cutoff at full second */
402 while (0 < values->values_num && values->values[values->values_num - 1].timestamp.sec == first_timestamp)
403 {
404 values->values_num--;
405 zbx_history_record_clear(&values->values[values->values_num], value_type);
406 left++;
407 }
408
409 offset = values->values_num;
410
411 if (FAIL == zbx_history_get_values(itemid, value_type, range_start, left, first_timestamp, values))
412 return FAIL;
413
414 /* returned less than requested - all values are read */
415 if (left > values->values_num - offset)
416 return SUCCEED;
417
418 first_timestamp = values->values[values->values_num - 1].timestamp.sec;
419 }
420
421 /* drop the first (oldest) second to ensure range cutoff at full second */
422 while (0 < values->values_num && values->values[values->values_num - 1].timestamp.sec == first_timestamp)
423 {
424 values->values_num--;
425 zbx_history_record_clear(&values->values[values->values_num], value_type);
426 }
427
428 /* check if there are enough values matching the request range */
429
430 for (i = values_start; i < values->values_num; i++)
431 {
432 if (0 <= zbx_timespec_compare(ts, &values->values[i].timestamp))
433 count--;
434 }
435
436 if (0 >= count)
437 return SUCCEED;
438
439 /* re-read the first (oldest) second */
440 return zbx_history_get_values(itemid, value_type, first_timestamp - 1, 0, first_timestamp, values);
441 }
442
443 /******************************************************************************
444 * *
445 * Function: vc_db_get_values *
446 * *
447 * Purpose: get item history data for the specified time period directly from *
448 * database *
449 * *
450 * Parameters: itemid - [IN] the item id *
451 * value_type - [IN] the item value type *
452 * values - [OUT] the item history data stored time/value *
453 * pairs in descending order *
454 * seconds - [IN] the time period to retrieve data for *
455 * count - [IN] the number of history values to retrieve *
456 * ts - [IN] the period end timestamp *
457 * *
458 * Return value: SUCCEED - the item history data was retrieved successfully *
459 * FAIL - the item history data was not retrieved *
460 * *
461 ******************************************************************************/
vc_db_get_values(zbx_uint64_t itemid,int value_type,zbx_vector_history_record_t * values,int seconds,int count,const zbx_timespec_t * ts)462 static int vc_db_get_values(zbx_uint64_t itemid, int value_type, zbx_vector_history_record_t *values, int seconds,
463 int count, const zbx_timespec_t *ts)
464 {
465 int ret = FAIL, i, j, range_start;
466
467 if (0 == count)
468 {
469 /* read one more second to cover for possible nanosecond shift */
470 ret = vc_db_read_values_by_time(itemid, value_type, values, ts->sec - seconds, ts->sec);
471 }
472 else
473 {
474 /* read one more second to cover for possible nanosecond shift */
475 range_start = (0 == seconds ? 0 : ts->sec - seconds);
476 ret = vc_db_read_values_by_time_and_count(itemid, value_type, values, range_start, count, ts->sec, ts);
477 }
478
479 if (SUCCEED != ret)
480 return ret;
481
482 zbx_vector_history_record_sort(values, (zbx_compare_func_t)zbx_history_record_compare_desc_func);
483
484 /* History backend returns values by full second intervals. With nanosecond resolution */
485 /* some of returned values might be outside the requested range, for example: */
486 /* returned values: |.o...o..o.|.o...o..o.|.o...o..o.|.o...o..o.| */
487 /* request range: \_______________________________/ */
488 /* Check if there are any values before and past the requested range and remove them. */
489
490 /* find the last value with timestamp less or equal to the requested range end point */
491 for (i = 0; i < values->values_num; i++)
492 {
493 if (0 >= zbx_timespec_compare(&values->values[i].timestamp, ts))
494 break;
495 }
496
497 /* all values are past requested range (timestamp greater than requested), return empty vector */
498 if (i == values->values_num)
499 {
500 vc_history_record_vector_clean(values, value_type);
501 return SUCCEED;
502 }
503
504 /* remove values with timestamp greater than the requested */
505 if (0 != i)
506 {
507 for (j = 0; j < i; j++)
508 zbx_history_record_clear(&values->values[j], value_type);
509
510 for (j = 0; i < values->values_num; i++, j++)
511 values->values[j] = values->values[i];
512
513 values->values_num = j;
514 }
515
516 /* for count based requests remove values exceeding requested count */
517 if (0 != count)
518 {
519 while (count < values->values_num)
520 zbx_history_record_clear(&values->values[--values->values_num], value_type);
521 }
522
523 /* for time based requests remove values with timestamp outside requested range */
524 if (0 != seconds)
525 {
526 zbx_timespec_t start = {ts->sec - seconds, ts->ns};
527
528 while (0 < values->values_num &&
529 0 >= zbx_timespec_compare(&values->values[values->values_num - 1].timestamp, &start))
530 {
531 zbx_history_record_clear(&values->values[--values->values_num], value_type);
532 }
533 }
534
535 return SUCCEED;
536 }
537
538 /******************************************************************************************************************
539 * *
540 * Common API *
541 * *
542 ******************************************************************************************************************/
543
544 /******************************************************************************
545 * *
546 * String pool definitions & functions *
547 * *
548 ******************************************************************************/
549
550 #define REFCOUNT_FIELD_SIZE sizeof(zbx_uint32_t)
551
vc_strpool_hash_func(const void * data)552 static zbx_hash_t vc_strpool_hash_func(const void *data)
553 {
554 return ZBX_DEFAULT_STRING_HASH_FUNC((char *)data + REFCOUNT_FIELD_SIZE);
555 }
556
vc_strpool_compare_func(const void * d1,const void * d2)557 static int vc_strpool_compare_func(const void *d1, const void *d2)
558 {
559 return strcmp((char *)d1 + REFCOUNT_FIELD_SIZE, (char *)d2 + REFCOUNT_FIELD_SIZE);
560 }
561
562 /******************************************************************************
563 * *
564 * Function: vc_item_weight_compare_func *
565 * *
566 * Purpose: compares two item weight data structures by their 'weight' *
567 * *
568 * Parameters: d1 - [IN] the first item weight data structure *
569 * d2 - [IN] the second item weight data structure *
570 * *
571 ******************************************************************************/
vc_item_weight_compare_func(const zbx_vc_item_weight_t * d1,const zbx_vc_item_weight_t * d2)572 static int vc_item_weight_compare_func(const zbx_vc_item_weight_t *d1, const zbx_vc_item_weight_t *d2)
573 {
574 ZBX_RETURN_IF_NOT_EQUAL(d1->weight, d2->weight);
575
576 return 0;
577 }
578
579 /******************************************************************************
580 * *
581 * Function: vc_history_logfree *
582 * *
583 * Purpose: frees history log and all resources allocated for it *
584 * *
585 * Parameters: log - [IN] the history log to free *
586 * *
587 ******************************************************************************/
vc_history_logfree(zbx_log_value_t * log)588 static void vc_history_logfree(zbx_log_value_t *log)
589 {
590 zbx_free(log->source);
591 zbx_free(log->value);
592 zbx_free(log);
593 }
594
595 /******************************************************************************
596 * *
597 * Function: vc_history_logdup *
598 * *
599 * Purpose: duplicates history log by allocating necessary resources and *
600 * copying the target log values. *
601 * *
602 * Parameters: log - [IN] the history log to duplicate *
603 * *
604 * Return value: the duplicated history log *
605 * *
606 ******************************************************************************/
vc_history_logdup(const zbx_log_value_t * log)607 static zbx_log_value_t *vc_history_logdup(const zbx_log_value_t *log)
608 {
609 zbx_log_value_t *plog;
610
611 plog = (zbx_log_value_t *)zbx_malloc(NULL, sizeof(zbx_log_value_t));
612
613 plog->timestamp = log->timestamp;
614 plog->logeventid = log->logeventid;
615 plog->severity = log->severity;
616 plog->source = (NULL == log->source ? NULL : zbx_strdup(NULL, log->source));
617 plog->value = zbx_strdup(NULL, log->value);
618
619 return plog;
620 }
621
622 /******************************************************************************
623 * *
624 * Function: vc_history_record_vector_clean *
625 * *
626 * Purpose: releases resources allocated to store history records *
627 * *
628 * Parameters: vector - [IN] the history record vector *
629 * value_type - [IN] the type of vector values *
630 * *
631 ******************************************************************************/
vc_history_record_vector_clean(zbx_vector_history_record_t * vector,int value_type)632 static void vc_history_record_vector_clean(zbx_vector_history_record_t *vector, int value_type)
633 {
634 int i;
635
636 switch (value_type)
637 {
638 case ITEM_VALUE_TYPE_STR:
639 case ITEM_VALUE_TYPE_TEXT:
640 for (i = 0; i < vector->values_num; i++)
641 zbx_free(vector->values[i].value.str);
642
643 break;
644 case ITEM_VALUE_TYPE_LOG:
645 for (i = 0; i < vector->values_num; i++)
646 vc_history_logfree(vector->values[i].value.log);
647 }
648
649 zbx_vector_history_record_clear(vector);
650 }
651
652 /******************************************************************************
653 * *
654 * Function: vc_update_statistics *
655 * *
656 * Purpose: updates cache and item statistics *
657 * *
658 * Parameters: item - [IN] the item (optional) *
659 * hits - [IN] the number of hits to add *
660 * misses - [IN] the number of misses to add *
661 * *
662 * Comments: The misses are added only to cache statistics, while hits are *
663 * added to both - item and cache statistics. *
664 * *
665 ******************************************************************************/
vc_update_statistics(zbx_vc_item_t * item,int hits,int misses,int now)666 static void vc_update_statistics(zbx_vc_item_t *item, int hits, int misses, int now)
667 {
668 if (NULL != item)
669 {
670 int hour;
671
672 item->hits += hits;
673 item->last_accessed = now;
674
675 hour = item->last_accessed / SEC_PER_HOUR;
676 if (hour != item->hour)
677 {
678 item->last_hourly_num = item->hourly_num;
679 item->hourly_num = 0;
680 item->hour = hour;
681 }
682
683 if (hits + misses > item->hourly_num)
684 item->hourly_num = hits + misses;
685 }
686
687 if (ZBX_VC_ENABLED == vc_state)
688 {
689 vc_cache->hits += hits;
690 vc_cache->misses += misses;
691 }
692 }
693
694 /******************************************************************************
695 * *
696 * Function: vc_compare_items_by_total_values *
697 * *
698 * Purpose: is used to sort items by value count in descending order *
699 * *
700 ******************************************************************************/
vc_compare_items_by_total_values(const void * d1,const void * d2)701 static int vc_compare_items_by_total_values(const void *d1, const void *d2)
702 {
703 zbx_vc_item_t *c1 = *(zbx_vc_item_t **)d1;
704 zbx_vc_item_t *c2 = *(zbx_vc_item_t **)d2;
705
706 ZBX_RETURN_IF_NOT_EQUAL(c2->values_total, c1->values_total);
707
708 return 0;
709 }
710
711 /******************************************************************************
712 * *
713 * Function: vc_dump_items_statistics *
714 * *
715 * Purpose: find out items responsible for low memory *
716 * *
717 ******************************************************************************/
vc_dump_items_statistics(void)718 static void vc_dump_items_statistics(void)
719 {
720 zbx_vc_item_t *item;
721 zbx_hashset_iter_t iter;
722 int i, total = 0, limit;
723 zbx_vector_ptr_t items;
724
725 zabbix_log(LOG_LEVEL_WARNING, "=== most used items statistics for value cache ===");
726
727 zbx_vector_ptr_create(&items);
728
729 zbx_hashset_iter_reset(&vc_cache->items, &iter);
730
731 while (NULL != (item = (zbx_vc_item_t *)zbx_hashset_iter_next(&iter)))
732 {
733 zbx_vector_ptr_append(&items, item);
734 total += item->values_total;
735 }
736
737 zbx_vector_ptr_sort(&items, vc_compare_items_by_total_values);
738
739 for (i = 0, limit = MIN(items.values_num, ZBX_VC_LOW_MEMORY_ITEM_PRINT_LIMIT); i < limit; i++)
740 {
741 item = (zbx_vc_item_t *)items.values[i];
742
743 zabbix_log(LOG_LEVEL_WARNING, "itemid:" ZBX_FS_UI64 " active range:%d hits:" ZBX_FS_UI64 " count:%d"
744 " perc:" ZBX_FS_DBL "%%", item->itemid, item->active_range, item->hits,
745 item->values_total, 100 * (double)item->values_total / total);
746 }
747
748 zbx_vector_ptr_destroy(&items);
749
750 zabbix_log(LOG_LEVEL_WARNING, "==================================================");
751 }
752
753 /******************************************************************************
754 * *
755 * Function: vc_warn_low_memory *
756 * *
757 * Purpose: logs low memory warning *
758 * *
759 * Comments: The low memory warning is written to log every 5 minutes when *
760 * cache is working in the low memory mode. *
761 * *
762 ******************************************************************************/
vc_warn_low_memory(void)763 static void vc_warn_low_memory(void)
764 {
765 int now;
766
767 now = time(NULL);
768
769 if (now - vc_cache->mode_time > ZBX_VC_LOW_MEMORY_RESET_PERIOD)
770 {
771 vc_cache->mode = ZBX_VC_MODE_NORMAL;
772 vc_cache->mode_time = now;
773
774 zabbix_log(LOG_LEVEL_WARNING, "value cache has been switched from low memory to normal operation mode");
775 }
776 else if (now - vc_cache->last_warning_time > ZBX_VC_LOW_MEMORY_WARNING_PERIOD)
777 {
778 vc_cache->last_warning_time = now;
779 vc_dump_items_statistics();
780 zbx_mem_dump_stats(LOG_LEVEL_WARNING, vc_mem);
781
782 zabbix_log(LOG_LEVEL_WARNING, "value cache is fully used: please increase ValueCacheSize"
783 " configuration parameter");
784 }
785 }
786
787 /******************************************************************************
788 * *
789 * Function: vc_release_unused_items *
790 * *
791 * Purpose: frees space in cache by dropping items not accessed for more than *
792 * 24 hours *
793 * *
794 * Parameters: source_item - [IN] the item requesting more space to store its *
795 * data *
796 * *
797 * Return value: number of bytes freed *
798 * *
799 ******************************************************************************/
vc_release_unused_items(const zbx_vc_item_t * source_item)800 static size_t vc_release_unused_items(const zbx_vc_item_t *source_item)
801 {
802 int timestamp;
803 zbx_hashset_iter_t iter;
804 zbx_vc_item_t *item;
805 size_t freed = 0;
806
807 if (NULL == vc_cache)
808 return freed;
809
810 timestamp = time(NULL) - ZBX_VC_ITEM_EXPIRE_PERIOD;
811
812 zbx_hashset_iter_reset(&vc_cache->items, &iter);
813
814 while (NULL != (item = (zbx_vc_item_t *)zbx_hashset_iter_next(&iter)))
815 {
816 if (0 != item->last_accessed && item->last_accessed < timestamp && source_item != item)
817 {
818 freed += vch_item_free_cache(item) + sizeof(zbx_vc_item_t);
819 zbx_hashset_iter_remove(&iter);
820 }
821 }
822
823 return freed;
824 }
825
826 /******************************************************************************
827 * *
828 * Function: zbx_vc_housekeeping_value_cache *
829 * *
830 * Purpose: release unused items from value cache *
831 * *
832 * Comments: If unused items are not cleared from value cache periodically *
833 * then they will only be cleared when value cache is full, see *
834 * vc_release_space(). *
835 * *
836 ******************************************************************************/
zbx_vc_housekeeping_value_cache(void)837 void zbx_vc_housekeeping_value_cache(void)
838 {
839 if (ZBX_VC_DISABLED == vc_state)
840 return;
841
842 WRLOCK_CACHE;
843 vc_release_unused_items(NULL);
844 UNLOCK_CACHE;
845 }
846
847 /******************************************************************************
848 * *
849 * Function: vc_release_space *
850 * *
851 * Purpose: frees space in cache to store the specified number of bytes by *
852 * dropping the least accessed items *
853 * *
854 * Parameters: item - [IN] the item requesting more space to store its data *
855 * space - [IN] the number of bytes to free *
856 * *
857 * Comments: The caller item must not be removed from cache to avoid *
858 * complications (ie - checking if item still is in cache every *
859 * time after calling vc_free_space() function). *
860 * vc_free_space() attempts to free at least min_free_request *
861 * bytes of space to reduce number of space release requests. *
862 * *
863 ******************************************************************************/
vc_release_space(zbx_vc_item_t * source_item,size_t space)864 static void vc_release_space(zbx_vc_item_t *source_item, size_t space)
865 {
866 zbx_hashset_iter_t iter;
867 zbx_vc_item_t *item;
868 int i;
869 size_t freed;
870 zbx_vector_vc_itemweight_t items;
871
872 /* reserve at least min_free_request bytes to avoid spamming with free space requests */
873 if (space < vc_cache->min_free_request)
874 space = vc_cache->min_free_request;
875
876 /* first remove items with the last accessed time older than a day */
877 if ((freed = vc_release_unused_items(source_item)) >= space)
878 return;
879
880 /* failed to free enough space by removing old items, entering low memory mode */
881 vc_cache->mode = ZBX_VC_MODE_LOWMEM;
882 vc_cache->mode_time = time(NULL);
883
884 vc_warn_low_memory();
885
886 /* remove items with least hits/size ratio */
887 zbx_vector_vc_itemweight_create(&items);
888
889 zbx_hashset_iter_reset(&vc_cache->items, &iter);
890
891 while (NULL != (item = (zbx_vc_item_t *)zbx_hashset_iter_next(&iter)))
892 {
893 /* don't remove the item that requested the space and also keep */
894 /* items currently being accessed */
895 if (item != source_item)
896 {
897 zbx_vc_item_weight_t weight = {.item = item};
898
899 if (0 < item->values_total)
900 weight.weight = (double)item->hits / item->values_total;
901
902 zbx_vector_vc_itemweight_append_ptr(&items, &weight);
903 }
904 }
905
906 zbx_vector_vc_itemweight_sort(&items, (zbx_compare_func_t)vc_item_weight_compare_func);
907
908 for (i = 0; i < items.values_num && freed < space; i++)
909 {
910 item = items.values[i].item;
911
912 freed += vch_item_free_cache(item) + sizeof(zbx_vc_item_t);
913 zbx_hashset_remove_direct(&vc_cache->items, item);
914 }
915 zbx_vector_vc_itemweight_destroy(&items);
916 }
917
918 /******************************************************************************
919 * *
920 * Function: vc_history_record_copy *
921 * *
922 * Purpose: copies history value *
923 * *
924 * Parameters: dst - [OUT] a pointer to the destination value *
925 * src - [IN] a pointer to the source value *
926 * value_type - [IN] the value type (see ITEM_VALUE_TYPE_* defs) *
927 * *
928 * Comments: Additional memory is allocated to store string, text and log *
929 * value contents. This memory must be freed by the caller. *
930 * *
931 ******************************************************************************/
vc_history_record_copy(zbx_history_record_t * dst,const zbx_history_record_t * src,int value_type)932 static void vc_history_record_copy(zbx_history_record_t *dst, const zbx_history_record_t *src, int value_type)
933 {
934 dst->timestamp = src->timestamp;
935
936 switch (value_type)
937 {
938 case ITEM_VALUE_TYPE_STR:
939 case ITEM_VALUE_TYPE_TEXT:
940 dst->value.str = zbx_strdup(NULL, src->value.str);
941 break;
942 case ITEM_VALUE_TYPE_LOG:
943 dst->value.log = vc_history_logdup(src->value.log);
944 break;
945 default:
946 dst->value = src->value;
947 }
948 }
949
950 /******************************************************************************
951 * *
952 * Function: vc_history_record_vector_append *
953 * *
954 * Purpose: appends the specified value to value vector *
955 * *
956 * Parameters: vector - [IN/OUT] the value vector *
957 * value_type - [IN] the type of value to append *
958 * value - [IN] the value to append *
959 * *
960 * Comments: Additional memory is allocated to store string, text and log *
961 * value contents. This memory must be freed by the caller. *
962 * *
963 ******************************************************************************/
vc_history_record_vector_append(zbx_vector_history_record_t * vector,int value_type,zbx_history_record_t * value)964 static void vc_history_record_vector_append(zbx_vector_history_record_t *vector, int value_type,
965 zbx_history_record_t *value)
966 {
967 zbx_history_record_t record;
968
969 vc_history_record_copy(&record, value, value_type);
970 zbx_vector_history_record_append_ptr(vector, &record);
971 }
972
973 /******************************************************************************
974 * *
975 * Function: vc_item_malloc *
976 * *
977 * Purpose: allocate cache memory to store item's resources *
978 * *
979 * Parameters: item - [IN] the item *
980 * size - [IN] the number of bytes to allocate *
981 * *
982 * Return value: The pointer to allocated memory or NULL if there is not *
983 * enough shared memory. *
984 * *
985 * Comments: If allocation fails this function attempts to free the required *
986 * space in cache by calling vc_free_space() and tries again. If it *
987 * still fails a NULL value is returned. *
988 * *
989 ******************************************************************************/
vc_item_malloc(zbx_vc_item_t * item,size_t size)990 static void *vc_item_malloc(zbx_vc_item_t *item, size_t size)
991 {
992 char *ptr;
993
994 if (NULL == (ptr = (char *)__vc_mem_malloc_func(NULL, size)))
995 {
996 /* If failed to allocate required memory, try to free space in */
997 /* cache and allocate again. If there still is not enough space - */
998 /* return NULL as failure. */
999 vc_release_space(item, size);
1000 ptr = (char *)__vc_mem_malloc_func(NULL, size);
1001 }
1002
1003 return ptr;
1004 }
1005
1006 /******************************************************************************
1007 * *
1008 * Function: vc_item_strdup *
1009 * *
1010 * Purpose: copies string to the cache memory *
1011 * *
1012 * Parameters: item - [IN] the item *
1013 * str - [IN] the string to copy *
1014 * *
1015 * Return value: The pointer to the copied string or NULL if there was not *
1016 * enough space in cache. *
1017 * *
1018 * Comments: If the string pool already contains matching string, then its *
1019 * reference counter is incremented and the string returned. *
1020 * *
1021 * Otherwise cache memory is allocated to store the specified *
1022 * string. If the allocation fails this function attempts to free *
1023 * the required space in cache by calling vc_release_space() and *
1024 * tries again. If it still fails then a NULL value is returned. *
1025 * *
1026 ******************************************************************************/
vc_item_strdup(zbx_vc_item_t * item,const char * str)1027 static char *vc_item_strdup(zbx_vc_item_t *item, const char *str)
1028 {
1029 void *ptr;
1030
1031 ptr = zbx_hashset_search(&vc_cache->strpool, str - REFCOUNT_FIELD_SIZE);
1032
1033 if (NULL == ptr)
1034 {
1035 int tries = 0;
1036 size_t len;
1037
1038 len = strlen(str) + 1;
1039
1040 while (NULL == (ptr = zbx_hashset_insert_ext(&vc_cache->strpool, str - REFCOUNT_FIELD_SIZE,
1041 REFCOUNT_FIELD_SIZE + len, REFCOUNT_FIELD_SIZE)))
1042 {
1043 /* If there is not enough space - free enough to store string + hashset entry overhead */
1044 /* and try inserting one more time. If it fails again, then fail the function. */
1045 if (0 == tries++)
1046 vc_release_space(item, len + REFCOUNT_FIELD_SIZE + sizeof(ZBX_HASHSET_ENTRY_T));
1047 else
1048 return NULL;
1049 }
1050
1051 *(zbx_uint32_t *)ptr = 0;
1052 }
1053
1054 (*(zbx_uint32_t *)ptr)++;
1055
1056 return (char *)ptr + REFCOUNT_FIELD_SIZE;
1057 }
1058
1059 /******************************************************************************
1060 * *
1061 * Function: vc_item_strfree *
1062 * *
1063 * Purpose: removes string from cache string pool *
1064 * *
1065 * Parameters: str - [IN] the string to remove *
1066 * *
1067 * Return value: the number of bytes freed *
1068 * *
1069 * Comments: This function decrements the string reference counter and *
1070 * removes it from the string pool when counter becomes zero. *
1071 * *
1072 * Note - only strings created with vc_item_strdup() function must *
1073 * be freed with vc_item_strfree(). *
1074 * *
1075 ******************************************************************************/
vc_item_strfree(char * str)1076 static size_t vc_item_strfree(char *str)
1077 {
1078 size_t freed = 0;
1079
1080 if (NULL != str)
1081 {
1082 void *ptr = str - REFCOUNT_FIELD_SIZE;
1083
1084 if (0 == --(*(zbx_uint32_t *)ptr))
1085 {
1086 freed = strlen(str) + REFCOUNT_FIELD_SIZE + 1;
1087 zbx_hashset_remove_direct(&vc_cache->strpool, ptr);
1088 }
1089 }
1090
1091 return freed;
1092 }
1093
1094 /******************************************************************************
1095 * *
1096 * Function: vc_item_logdup *
1097 * *
1098 * Purpose: copies log value to the cache memory *
1099 * *
1100 * Parameters: item - [IN] the item *
1101 * log - [IN] the log value to copy *
1102 * *
1103 * Return value: The pointer to the copied log value or NULL if there was *
1104 * not enough space in cache. *
1105 * *
1106 * Comments: Cache memory is allocated to store the log value. If the *
1107 * allocation fails this function attempts to free the required *
1108 * space in cache by calling vc_release_space() and tries again. *
1109 * If it still fails then a NULL value is returned. *
1110 * *
1111 ******************************************************************************/
vc_item_logdup(zbx_vc_item_t * item,const zbx_log_value_t * log)1112 static zbx_log_value_t *vc_item_logdup(zbx_vc_item_t *item, const zbx_log_value_t *log)
1113 {
1114 zbx_log_value_t *plog = NULL;
1115
1116 if (NULL == (plog = (zbx_log_value_t *)vc_item_malloc(item, sizeof(zbx_log_value_t))))
1117 return NULL;
1118
1119 plog->timestamp = log->timestamp;
1120 plog->logeventid = log->logeventid;
1121 plog->severity = log->severity;
1122
1123 if (NULL != log->source)
1124 {
1125 if (NULL == (plog->source = vc_item_strdup(item, log->source)))
1126 goto fail;
1127 }
1128 else
1129 plog->source = NULL;
1130
1131 if (NULL == (plog->value = vc_item_strdup(item, log->value)))
1132 goto fail;
1133
1134 return plog;
1135 fail:
1136 vc_item_strfree(plog->source);
1137
1138 __vc_mem_free_func(plog);
1139
1140 return NULL;
1141 }
1142
1143 /******************************************************************************
1144 * *
1145 * Function: vc_item_logfree *
1146 * *
1147 * Purpose: removes log resource from cache memory *
1148 * *
1149 * Parameters: str - [IN] the log to remove *
1150 * *
1151 * Return value: the number of bytes freed *
1152 * *
1153 * Comments: Note - only logs created with vc_item_logdup() function must *
1154 * be freed with vc_item_logfree(). *
1155 * *
1156 ******************************************************************************/
vc_item_logfree(zbx_log_value_t * log)1157 static size_t vc_item_logfree(zbx_log_value_t *log)
1158 {
1159 size_t freed = 0;
1160
1161 if (NULL != log)
1162 {
1163 freed += vc_item_strfree(log->source);
1164 freed += vc_item_strfree(log->value);
1165
1166 __vc_mem_free_func(log);
1167 freed += sizeof(zbx_log_value_t);
1168 }
1169
1170 return freed;
1171 }
1172
1173 /******************************************************************************
1174 * *
1175 * Function: vc_item_free_values *
1176 * *
1177 * Purpose: frees cache resources of the specified item value range *
1178 * *
1179 * Parameters: item - [IN] the item *
1180 * values - [IN] the target value array *
1181 * first - [IN] the first value to free *
1182 * last - [IN] the last value to free *
1183 * *
1184 * Return value: the number of bytes freed *
1185 * *
1186 ******************************************************************************/
vc_item_free_values(zbx_vc_item_t * item,zbx_history_record_t * values,int first,int last)1187 static size_t vc_item_free_values(zbx_vc_item_t *item, zbx_history_record_t *values, int first, int last)
1188 {
1189 size_t freed = 0;
1190 int i;
1191
1192 switch (item->value_type)
1193 {
1194 case ITEM_VALUE_TYPE_STR:
1195 case ITEM_VALUE_TYPE_TEXT:
1196 for (i = first; i <= last; i++)
1197 freed += vc_item_strfree(values[i].value.str);
1198 break;
1199 case ITEM_VALUE_TYPE_LOG:
1200 for (i = first; i <= last; i++)
1201 freed += vc_item_logfree(values[i].value.log);
1202 break;
1203 }
1204
1205 item->values_total -= (last - first + 1);
1206
1207 return freed;
1208 }
1209
1210 /******************************************************************************
1211 * *
1212 * Function: vc_remove_item *
1213 * *
1214 * Purpose: removes item from cache and frees resources allocated for it *
1215 * *
1216 * Parameters: item - [IN] the item *
1217 * *
1218 ******************************************************************************/
vc_remove_item(zbx_vc_item_t * item)1219 static void vc_remove_item(zbx_vc_item_t *item)
1220 {
1221 vch_item_free_cache(item);
1222 zbx_hashset_remove_direct(&vc_cache->items, item);
1223 }
1224
1225 /******************************************************************************
1226 * *
1227 * Function: vc_remove_item_by_id *
1228 * *
1229 * Purpose: removes item from cache and frees resources allocated for it *
1230 * *
1231 * Parameters: itemid - [IN] the item identifier *
1232 * *
1233 ******************************************************************************/
vc_remove_item_by_id(zbx_uint64_t itemid)1234 static void vc_remove_item_by_id(zbx_uint64_t itemid)
1235 {
1236 zbx_vc_item_t *item;
1237
1238 if (NULL == (item = (zbx_vc_item_t *)zbx_hashset_search(&vc_cache->items, &itemid)))
1239 return;
1240
1241 vch_item_free_cache(item);
1242 zbx_hashset_remove_direct(&vc_cache->items, item);
1243 }
1244 /******************************************************************************
1245 * *
1246 * Function: vc_item_update_db_cached_from *
1247 * *
1248 * Purpose: updates the timestamp from which the item is being cached *
1249 * *
1250 * Parameters: item - [IN] the item *
1251 * timestamp - [IN] the timestamp from which all item values are *
1252 * guaranteed to be cached *
1253 * *
1254 ******************************************************************************/
vc_item_update_db_cached_from(zbx_vc_item_t * item,int timestamp)1255 static void vc_item_update_db_cached_from(zbx_vc_item_t *item, int timestamp)
1256 {
1257 if (0 == item->db_cached_from || timestamp < item->db_cached_from)
1258 item->db_cached_from = timestamp;
1259 }
1260
1261 /******************************************************************************************************************
1262 * *
1263 * History storage API *
1264 * *
1265 ******************************************************************************************************************/
1266 /*
1267 * The value cache caches all values from the largest request range to
1268 * the current time. The history data are stored in variable size chunks
1269 * as illustrated in the following diagram:
1270 *
1271 * .----------------.
1272 * | zbx_vc_cache_t |
1273 * |----------------| .---------------.
1274 * | items |----->| zbx_vc_item_t |-.
1275 * '----------------' |---------------| |-.
1276 * .-----------------------| tail | | |
1277 * | .---| head | | |
1278 * | | '---------------' | |
1279 * | | '---------------' |
1280 * | | '---------------'
1281 * | |
1282 * | '-------------------------------------------------.
1283 * | |
1284 * | .----------------. |
1285 * '->| zbx_vc_chunk_t |<-. |
1286 * |----------------| | .----------------. |
1287 * | next |---->| zbx_vc_chunk_t |<-. |
1288 * | prev | | |----------------| | .----------------. |
1289 * '----------------' | | next |---->| zbx_vc_chunk_t |<--'
1290 * '--| prev | | |----------------|
1291 * '----------------' | | next |
1292 * '--| prev |
1293 * '----------------'
1294 *
1295 * The history values are stored in a double linked list of data chunks, holding
1296 * variable number of records (depending on largest request size).
1297 *
1298 * After adding a new chunk, the older chunks (outside the largest request
1299 * range) are automatically removed from cache.
1300 */
1301
1302 /******************************************************************************
1303 * *
1304 * Function: vch_item_update_range *
1305 * *
1306 * Purpose: updates item range with current request range *
1307 * *
1308 * Parameters: item - [IN] the item *
1309 * range - [IN] the request range *
1310 * now - [IN] the current timestamp *
1311 * *
1312 ******************************************************************************/
vch_item_update_range(zbx_vc_item_t * item,int range,int now)1313 static void vch_item_update_range(zbx_vc_item_t *item, int range, int now)
1314 {
1315 int hour, diff;
1316
1317 if (VC_MIN_RANGE > range)
1318 range = VC_MIN_RANGE;
1319
1320 if (item->daily_range < range)
1321 item->daily_range = range;
1322
1323 hour = (now / SEC_PER_HOUR) & 0xff;
1324
1325 if (0 > (diff = hour - item->range_sync_hour))
1326 diff += 0xff;
1327
1328 if (item->active_range < item->daily_range || ZBX_VC_RANGE_SYNC_PERIOD < diff)
1329 {
1330 item->active_range = item->daily_range;
1331 item->daily_range = range;
1332 item->range_sync_hour = hour;
1333 }
1334 }
1335
1336 /******************************************************************************
1337 * *
1338 * Function: vch_item_chunk_slot_count *
1339 * *
1340 * Purpose: calculates optimal number of slots for an item data chunk *
1341 * *
1342 * Parameters: item - [IN] the item *
1343 * values_new - [IN] the number of values to be added *
1344 * *
1345 * Return value: the number of slots for a new item data chunk *
1346 * *
1347 * Comments: From size perspective the optimal slot count per chunk is *
1348 * approximately square root of the number of cached values. *
1349 * Still creating too many chunks might affect timeshift request *
1350 * performance, so don't try creating more than 32 chunks unless *
1351 * the calculated slot count exceeds the maximum limit. *
1352 * *
1353 ******************************************************************************/
vch_item_chunk_slot_count(zbx_vc_item_t * item,int values_new)1354 static int vch_item_chunk_slot_count(zbx_vc_item_t *item, int values_new)
1355 {
1356 int nslots, values;
1357
1358 values = item->values_total + values_new;
1359
1360 nslots = zbx_isqrt32(values);
1361
1362 if ((values + nslots - 1) / nslots + 1 > 32)
1363 nslots = values / 32;
1364
1365 if (nslots > (int)ZBX_VC_MAX_CHUNK_RECORDS)
1366 nslots = ZBX_VC_MAX_CHUNK_RECORDS;
1367 if (nslots < (int)ZBX_VC_MIN_CHUNK_RECORDS)
1368 nslots = ZBX_VC_MIN_CHUNK_RECORDS;
1369
1370 return nslots;
1371 }
1372
1373 /******************************************************************************
1374 * *
1375 * Function: vch_item_add_chunk *
1376 * *
1377 * Purpose: adds a new data chunk at the end of item's history data list *
1378 * *
1379 * Parameters: item - [IN/OUT] the item to add chunk to *
1380 * nslots - [IN] the number of slots in the new chunk *
1381 * insert_before - [IN] the target chunk before which the new *
1382 * chunk must be inserted. If this value is NULL *
1383 * then the new chunk is appended at the end of *
1384 * chunk list (making it the newest chunk). *
1385 * *
1386 * Return value: SUCCEED - the chunk was added successfully *
1387 * FAIL - failed to create a new chunk (not enough memory) *
1388 * *
1389 ******************************************************************************/
vch_item_add_chunk(zbx_vc_item_t * item,int nslots,zbx_vc_chunk_t * insert_before)1390 static int vch_item_add_chunk(zbx_vc_item_t *item, int nslots, zbx_vc_chunk_t *insert_before)
1391 {
1392 zbx_vc_chunk_t *chunk;
1393 int chunk_size;
1394
1395 chunk_size = sizeof(zbx_vc_chunk_t) + sizeof(zbx_history_record_t) * (nslots - 1);
1396
1397 if (NULL == (chunk = (zbx_vc_chunk_t *)vc_item_malloc(item, chunk_size)))
1398 return FAIL;
1399
1400 memset(chunk, 0, sizeof(zbx_vc_chunk_t));
1401 chunk->slots_num = nslots;
1402
1403 chunk->next = insert_before;
1404
1405 if (NULL == insert_before)
1406 {
1407 chunk->prev = item->head;
1408
1409 if (NULL != item->head)
1410 item->head->next = chunk;
1411 else
1412 item->tail = chunk;
1413
1414 item->head = chunk;
1415 }
1416 else
1417 {
1418 chunk->prev = insert_before->prev;
1419 insert_before->prev = chunk;
1420
1421 if (item->tail == insert_before)
1422 item->tail = chunk;
1423 else
1424 chunk->prev->next = chunk;
1425 }
1426
1427 return SUCCEED;
1428 }
1429
1430 /******************************************************************************
1431 * *
1432 * Function: vch_chunk_find_last_value_before *
1433 * *
1434 * Purpose: find the index of the last value in chunk with timestamp less or *
1435 * equal to the specified timestamp. *
1436 * *
1437 * Parameters: chunk - [IN] the chunk *
1438 * ts - [IN] the target timestamp *
1439 * *
1440 * Return value: The index of the last value in chunk with timestamp less or *
1441 * equal to the specified timestamp. *
1442 * -1 is returned in the case of failure (meaning that all *
1443 * values have timestamps greater than the target timestamp). *
1444 * *
1445 ******************************************************************************/
vch_chunk_find_last_value_before(const zbx_vc_chunk_t * chunk,const zbx_timespec_t * ts)1446 static int vch_chunk_find_last_value_before(const zbx_vc_chunk_t *chunk, const zbx_timespec_t *ts)
1447 {
1448 int start = chunk->first_value, end = chunk->last_value, middle;
1449
1450 /* check if the last value timestamp is already greater or equal to the specified timestamp */
1451 if (0 >= zbx_timespec_compare(&chunk->slots[end].timestamp, ts))
1452 return end;
1453
1454 /* chunk contains only one value, which did not pass the above check, return failure */
1455 if (start == end)
1456 return -1;
1457
1458 /* perform value lookup using binary search */
1459 while (start != end)
1460 {
1461 middle = start + (end - start) / 2;
1462
1463 if (0 < zbx_timespec_compare(&chunk->slots[middle].timestamp, ts))
1464 {
1465 end = middle;
1466 continue;
1467 }
1468
1469 if (0 >= zbx_timespec_compare(&chunk->slots[middle + 1].timestamp, ts))
1470 {
1471 start = middle;
1472 continue;
1473 }
1474
1475 return middle;
1476 }
1477
1478 return -1;
1479 }
1480
1481 /******************************************************************************
1482 * *
1483 * Function: vch_item_get_last_value *
1484 * *
1485 * Purpose: gets the chunk and index of the last value with a timestamp less *
1486 * or equal to the specified timestamp *
1487 * *
1488 * Parameters: item - [IN] the item *
1489 * ts - [IN] the target timestamp *
1490 * (NULL - current time) *
1491 * pchunk - [OUT] the chunk containing the target value *
1492 * pindex - [OUT] the index of the target value *
1493 * *
1494 * Return value: SUCCEED - the last value was found successfully *
1495 * FAIL - all values in cache have timestamps greater than the *
1496 * target (timeshift) timestamp. *
1497 * *
1498 * Comments: If end_timestamp value is 0, then simply the last item value in *
1499 * cache is returned. *
1500 * *
1501 ******************************************************************************/
vch_item_get_last_value(const zbx_vc_item_t * item,const zbx_timespec_t * ts,zbx_vc_chunk_t ** pchunk,int * pindex)1502 static int vch_item_get_last_value(const zbx_vc_item_t *item, const zbx_timespec_t *ts, zbx_vc_chunk_t **pchunk,
1503 int *pindex)
1504 {
1505 zbx_vc_chunk_t *chunk = item->head;
1506 int index;
1507
1508 if (NULL == chunk)
1509 return FAIL;
1510
1511 index = chunk->last_value;
1512
1513 if (0 < zbx_timespec_compare(&chunk->slots[index].timestamp, ts))
1514 {
1515 while (0 < zbx_timespec_compare(&chunk->slots[chunk->first_value].timestamp, ts))
1516 {
1517 chunk = chunk->prev;
1518 /* there are no values for requested range, return failure */
1519 if (NULL == chunk)
1520 return FAIL;
1521 }
1522 index = vch_chunk_find_last_value_before(chunk, ts);
1523 }
1524
1525 *pchunk = chunk;
1526 *pindex = index;
1527
1528 return SUCCEED;
1529 }
1530
1531 /******************************************************************************
1532 * *
1533 * Function: vch_item_copy_value *
1534 * *
1535 * Purpose: copies value in the specified item's chunk slot *
1536 * *
1537 * Parameters: chunk - [IN/OUT] the target chunk *
1538 * index - [IN] the target slot *
1539 * source_value - [IN] the value to copy *
1540 * *
1541 * Return value: SUCCEED - the value was copied successfully *
1542 * FAIL - the value copying failed (not enough space for *
1543 * string, text or log type data) *
1544 * *
1545 * Comments: This function is used to copy data to cache. The contents of *
1546 * str, text and log type values are stored in cache string pool. *
1547 * *
1548 ******************************************************************************/
vch_item_copy_value(zbx_vc_item_t * item,zbx_vc_chunk_t * chunk,int index,const zbx_history_record_t * source_value)1549 static int vch_item_copy_value(zbx_vc_item_t *item, zbx_vc_chunk_t *chunk, int index,
1550 const zbx_history_record_t *source_value)
1551 {
1552 zbx_history_record_t *value;
1553 int ret = FAIL;
1554
1555 value = &chunk->slots[index];
1556
1557 switch (item->value_type)
1558 {
1559 case ITEM_VALUE_TYPE_STR:
1560 case ITEM_VALUE_TYPE_TEXT:
1561 if (NULL == (value->value.str = vc_item_strdup(item, source_value->value.str)))
1562 goto out;
1563 break;
1564 case ITEM_VALUE_TYPE_LOG:
1565 if (NULL == (value->value.log = vc_item_logdup(item, source_value->value.log)))
1566 goto out;
1567 break;
1568 default:
1569 value->value = source_value->value;
1570 }
1571 value->timestamp = source_value->timestamp;
1572
1573 ret = SUCCEED;
1574 out:
1575 return ret;
1576 }
1577
1578 /******************************************************************************
1579 * *
1580 * Function: vch_item_copy_values_at_tail *
1581 * *
1582 * Purpose: copies values at the beginning of item tail chunk *
1583 * *
1584 * Parameters: item - [IN/OUT] the target item *
1585 * values - [IN] the values to copy *
1586 * values_num - [IN] the number of values to copy *
1587 * *
1588 * Return value: SUCCEED - the values were copied successfully *
1589 * FAIL - the value copying failed (not enough space for *
1590 * string, text or log type data) *
1591 * *
1592 * Comments: This function is used to copy data to cache. The contents of *
1593 * str, text and log type values are stored in cache string pool. *
1594 * *
1595 ******************************************************************************/
vch_item_copy_values_at_tail(zbx_vc_item_t * item,const zbx_history_record_t * values,int values_num)1596 static int vch_item_copy_values_at_tail(zbx_vc_item_t *item, const zbx_history_record_t *values, int values_num)
1597 {
1598 int i, ret = FAIL, first_value = item->tail->first_value;
1599
1600 switch (item->value_type)
1601 {
1602 case ITEM_VALUE_TYPE_STR:
1603 case ITEM_VALUE_TYPE_TEXT:
1604 for (i = values_num - 1; i >= 0; i--)
1605 {
1606 zbx_history_record_t *value = &item->tail->slots[item->tail->first_value - 1];
1607
1608 if (NULL == (value->value.str = vc_item_strdup(item, values[i].value.str)))
1609 goto out;
1610
1611 value->timestamp = values[i].timestamp;
1612 item->tail->first_value--;
1613 }
1614 ret = SUCCEED;
1615
1616 break;
1617 case ITEM_VALUE_TYPE_LOG:
1618 for (i = values_num - 1; i >= 0; i--)
1619 {
1620 zbx_history_record_t *value = &item->tail->slots[item->tail->first_value - 1];
1621
1622 if (NULL == (value->value.log = vc_item_logdup(item, values[i].value.log)))
1623 goto out;
1624
1625 value->timestamp = values[i].timestamp;
1626 item->tail->first_value--;
1627 }
1628 ret = SUCCEED;
1629
1630 break;
1631 default:
1632 memcpy(&item->tail->slots[item->tail->first_value - values_num], values,
1633 values_num * sizeof(zbx_history_record_t));
1634 item->tail->first_value -= values_num;
1635 ret = SUCCEED;
1636 }
1637 out:
1638 item->values_total += first_value - item->tail->first_value;
1639
1640 return ret;
1641 }
1642
1643 /******************************************************************************
1644 * *
1645 * Function: vch_item_free_chunk *
1646 * *
1647 * Purpose: frees chunk and all resources allocated to store its values *
1648 * *
1649 * Parameters: item - [IN] the chunk owner item *
1650 * chunk - [IN] the chunk to free *
1651 * *
1652 * Return value: the number of bytes freed *
1653 * *
1654 ******************************************************************************/
vch_item_free_chunk(zbx_vc_item_t * item,zbx_vc_chunk_t * chunk)1655 static size_t vch_item_free_chunk(zbx_vc_item_t *item, zbx_vc_chunk_t *chunk)
1656 {
1657 size_t freed;
1658
1659 freed = sizeof(zbx_vc_chunk_t) + (chunk->slots_num - 1) * sizeof(zbx_history_record_t);
1660 freed += vc_item_free_values(item, chunk->slots, chunk->first_value, chunk->last_value);
1661
1662 __vc_mem_free_func(chunk);
1663
1664 return freed;
1665 }
1666
1667 /******************************************************************************
1668 * *
1669 * Function: vch_item_remove_chunk *
1670 * *
1671 * Purpose: removes item history data chunk *
1672 * *
1673 * Parameters: item - [IN ] the chunk owner item *
1674 * chunk - [IN] the chunk to remove *
1675 * *
1676 ******************************************************************************/
vch_item_remove_chunk(zbx_vc_item_t * item,zbx_vc_chunk_t * chunk)1677 static void vch_item_remove_chunk(zbx_vc_item_t *item, zbx_vc_chunk_t *chunk)
1678 {
1679 if (NULL != chunk->next)
1680 chunk->next->prev = chunk->prev;
1681
1682 if (NULL != chunk->prev)
1683 chunk->prev->next = chunk->next;
1684
1685 if (chunk == item->head)
1686 item->head = chunk->prev;
1687
1688 if (chunk == item->tail)
1689 item->tail = chunk->next;
1690
1691 vch_item_free_chunk(item, chunk);
1692 }
1693
1694 /******************************************************************************
1695 * *
1696 * Function: vch_item_clean_cache *
1697 * *
1698 * Purpose: removes item history data that are outside (older) the maximum *
1699 * request range *
1700 * *
1701 * Parameters: item - [IN] the target item *
1702 * *
1703 ******************************************************************************/
vch_item_clean_cache(zbx_vc_item_t * item)1704 static void vch_item_clean_cache(zbx_vc_item_t *item)
1705 {
1706 zbx_vc_chunk_t *next;
1707
1708 if (0 != item->active_range)
1709 {
1710 zbx_vc_chunk_t *tail = item->tail;
1711 zbx_vc_chunk_t *chunk = tail;
1712 int timestamp;
1713
1714 timestamp = time(NULL) - item->active_range;
1715
1716 /* try to remove chunks with all history values older than maximum request range */
1717 while (NULL != chunk && chunk->slots[chunk->last_value].timestamp.sec < timestamp &&
1718 chunk->slots[chunk->last_value].timestamp.sec !=
1719 item->head->slots[item->head->last_value].timestamp.sec)
1720 {
1721 /* don't remove the head chunk */
1722 if (NULL == (next = chunk->next))
1723 break;
1724
1725 /* Values with the same timestamps (seconds resolution) always should be either */
1726 /* kept in cache or removed together. There should not be a case when one of them */
1727 /* is in cache and the second is dropped. */
1728 /* Here we are handling rare case, when the last value of first chunk has the */
1729 /* same timestamp (seconds resolution) as the first value in the second chunk. */
1730 /* In this case increase the first value index of the next chunk until the first */
1731 /* value timestamp is greater. */
1732
1733 if (next->slots[next->first_value].timestamp.sec != next->slots[next->last_value].timestamp.sec)
1734 {
1735 while (next->slots[next->first_value].timestamp.sec ==
1736 chunk->slots[chunk->last_value].timestamp.sec)
1737 {
1738 vc_item_free_values(item, next->slots, next->first_value, next->first_value);
1739 next->first_value++;
1740 }
1741 }
1742
1743 /* set the database cached from timestamp to the last (oldest) removed value timestamp + 1 */
1744 item->db_cached_from = chunk->slots[chunk->last_value].timestamp.sec + 1;
1745
1746 vch_item_remove_chunk(item, chunk);
1747
1748 chunk = next;
1749 }
1750
1751 /* reset the status flags if data was removed from cache */
1752 if (tail != item->tail)
1753 item->status = 0;
1754 }
1755 }
1756
1757 /******************************************************************************
1758 * *
1759 * Function: vch_item_remove_values *
1760 * *
1761 * Purpose: removes item history data that are older than the specified *
1762 * timestamp *
1763 * *
1764 * Parameters: item - [IN] the target item *
1765 * timestamp - [IN] the timestamp (number of seconds since the *
1766 * Epoch) *
1767 * *
1768 ******************************************************************************/
vch_item_remove_values(zbx_vc_item_t * item,int timestamp)1769 static void vch_item_remove_values(zbx_vc_item_t *item, int timestamp)
1770 {
1771 zbx_vc_chunk_t *chunk = item->tail;
1772
1773 if (ZBX_ITEM_STATUS_CACHED_ALL == item->status)
1774 item->status = 0;
1775
1776 /* try to remove chunks with all history values older than the timestamp */
1777 while (NULL != chunk && chunk->slots[chunk->first_value].timestamp.sec < timestamp)
1778 {
1779 zbx_vc_chunk_t *next;
1780
1781 /* If chunk contains values with timestamp greater or equal - remove */
1782 /* only the values with less timestamp. Otherwise remove the while */
1783 /* chunk and check next one. */
1784 if (chunk->slots[chunk->last_value].timestamp.sec >= timestamp)
1785 {
1786 while (chunk->slots[chunk->first_value].timestamp.sec < timestamp)
1787 {
1788 vc_item_free_values(item, chunk->slots, chunk->first_value, chunk->first_value);
1789 chunk->first_value++;
1790 }
1791
1792 break;
1793 }
1794
1795 next = chunk->next;
1796 vch_item_remove_chunk(item, chunk);
1797 chunk = next;
1798 }
1799 }
1800
1801 /******************************************************************************
1802 * *
1803 * Function: vch_item_add_value_at_head *
1804 * *
1805 * Purpose: adds one item history value at the end of current item's history *
1806 * data *
1807 * *
1808 * Parameters: item - [IN] the item to add history data to *
1809 * value - [IN] the item history data value *
1810 * *
1811 * Return value: SUCCEED - the history data value was added successfully *
1812 * FAIL - failed to add history data value (not enough memory) *
1813 * *
1814 * Comments: In the case of failure the item will be removed from cache *
1815 * later. *
1816 * *
1817 ******************************************************************************/
vch_item_add_value_at_head(zbx_vc_item_t * item,const zbx_history_record_t * value)1818 static int vch_item_add_value_at_head(zbx_vc_item_t *item, const zbx_history_record_t *value)
1819 {
1820 int ret = FAIL, index, sindex, nslots = 0;
1821 zbx_vc_chunk_t *chunk, *schunk;
1822
1823 if (NULL != item->head &&
1824 0 < zbx_history_record_compare_asc_func(&item->head->slots[item->head->last_value], value))
1825 {
1826 if (0 < zbx_history_record_compare_asc_func(&item->tail->slots[item->tail->first_value], value))
1827 {
1828 /* If the added value has the same or older timestamp as the first value in cache */
1829 /* we can't add it to keep cache consistency. Additionally we must make sure no */
1830 /* values with matching timestamp seconds are kept in cache. */
1831 vch_item_remove_values(item, value->timestamp.sec + 1);
1832
1833 /* empty items must be removed to avoid situation when a new value is added to cache */
1834 /* while other values with matching timestamp seconds are not cached */
1835 if (NULL == item->head)
1836 goto out;
1837
1838 /* if the value is newer than the database cached from timestamp we must */
1839 /* adjust the cached from timestamp to exclude this value */
1840 if (item->db_cached_from <= value->timestamp.sec)
1841 item->db_cached_from = value->timestamp.sec + 1;
1842
1843 ret = SUCCEED;
1844 goto out;
1845 }
1846
1847 sindex = item->head->last_value;
1848 schunk = item->head;
1849
1850 if (0 == item->head->slots_num - item->head->last_value - 1)
1851 {
1852 if (FAIL == vch_item_add_chunk(item, vch_item_chunk_slot_count(item, 1), NULL))
1853 goto out;
1854 }
1855 else
1856 item->head->last_value++;
1857
1858 item->values_total++;
1859
1860 chunk = item->head;
1861 index = item->head->last_value;
1862
1863 do
1864 {
1865 chunk->slots[index] = schunk->slots[sindex];
1866
1867 chunk = schunk;
1868 index = sindex;
1869
1870 if (--sindex < schunk->first_value)
1871 {
1872 if (NULL == (schunk = schunk->prev))
1873 {
1874 memset(&chunk->slots[index], 0, sizeof(zbx_vc_chunk_t));
1875 THIS_SHOULD_NEVER_HAPPEN;
1876
1877 goto out;
1878 }
1879
1880 sindex = schunk->last_value;
1881 }
1882 }
1883 while (0 < zbx_timespec_compare(&schunk->slots[sindex].timestamp, &value->timestamp));
1884 }
1885 else
1886 {
1887 /* find the number of free slots on the right side in last (head) chunk */
1888 if (NULL != item->head)
1889 nslots = item->head->slots_num - item->head->last_value - 1;
1890
1891 if (0 == nslots)
1892 {
1893 if (FAIL == vch_item_add_chunk(item, vch_item_chunk_slot_count(item, 1), NULL))
1894 goto out;
1895 }
1896 else
1897 item->head->last_value++;
1898
1899 item->values_total++;
1900
1901 chunk = item->head;
1902 index = item->head->last_value;
1903 }
1904
1905 if (SUCCEED != vch_item_copy_value(item, chunk, index, value))
1906 goto out;
1907
1908 ret = SUCCEED;
1909 out:
1910 return ret;
1911 }
1912
1913 /******************************************************************************
1914 * *
1915 * Function: vch_item_add_values_at_tail *
1916 * *
1917 * Purpose: adds item history values at the beginning of current item's *
1918 * history data *
1919 * *
1920 * Parameters: item - [IN] the item to add history data to *
1921 * values - [IN] the item history data values *
1922 * num - [IN] the number of history data values to add *
1923 * *
1924 * Return value: SUCCEED - the history data values were added successfully *
1925 * FAIL - failed to add history data values (not enough memory) *
1926 * *
1927 * Comments: In the case of failure the item is removed from cache. *
1928 * Overlapping values (by timestamp seconds) are ignored. *
1929 * *
1930 ******************************************************************************/
vch_item_add_values_at_tail(zbx_vc_item_t * item,const zbx_history_record_t * values,int values_num)1931 static int vch_item_add_values_at_tail(zbx_vc_item_t *item, const zbx_history_record_t *values, int values_num)
1932 {
1933 int count = values_num, ret = FAIL;
1934
1935 /* skip values already added to the item cache by another process */
1936 if (NULL != item->tail)
1937 {
1938 int sec = item->tail->slots[item->tail->first_value].timestamp.sec;
1939
1940 while (--count >= 0 && values[count].timestamp.sec >= sec)
1941 ;
1942 ++count;
1943 }
1944
1945 while (0 != count)
1946 {
1947 int copy_slots, nslots = 0;
1948
1949 /* find the number of free slots on the left side in first (tail) chunk */
1950 if (NULL != item->tail)
1951 nslots = item->tail->first_value;
1952
1953 if (0 == nslots)
1954 {
1955 nslots = vch_item_chunk_slot_count(item, count);
1956
1957 if (FAIL == vch_item_add_chunk(item, nslots, item->tail))
1958 goto out;
1959
1960 item->tail->last_value = nslots - 1;
1961 item->tail->first_value = nslots;
1962 }
1963
1964 /* copy values to chunk */
1965 copy_slots = MIN(nslots, count);
1966 count -= copy_slots;
1967
1968 if (FAIL == vch_item_copy_values_at_tail(item, values + count, copy_slots))
1969 goto out;
1970 }
1971
1972 ret = SUCCEED;
1973 out:
1974 return ret;
1975 }
1976
1977 /******************************************************************************
1978 * *
1979 * Function: vch_item_cache_values_by_time *
1980 * *
1981 * Purpose: cache item history data for the specified time period *
1982 * *
1983 * Parameters: item - [IN] the item *
1984 * range_start - [IN] the interval start time *
1985 * *
1986 * Return value: >=0 - the number of values read from database *
1987 * FAIL - an error occurred while trying to cache values *
1988 * *
1989 * Comments: This function checks if the requested value range is cached and *
1990 * updates cache from database if necessary. *
1991 * *
1992 ******************************************************************************/
vch_item_cache_values_by_time(zbx_vc_item_t ** item,int range_start)1993 static int vch_item_cache_values_by_time(zbx_vc_item_t **item, int range_start)
1994 {
1995 int ret, range_end;
1996 zbx_vector_history_record_t records;
1997 zbx_uint64_t itemid;
1998 unsigned char value_type;
1999
2000 if (ZBX_ITEM_STATUS_CACHED_ALL == (*item)->status)
2001 return SUCCEED;
2002
2003 /* check if the requested period is in the cached range */
2004 if (0 != (*item)->db_cached_from && range_start >= (*item)->db_cached_from)
2005 return SUCCEED;
2006
2007 /* find if the cache should be updated to cover the required range */
2008 if (NULL != (*item)->tail)
2009 {
2010 /* we need to get item values before the first cached value, but not including it */
2011 range_end = (*item)->tail->slots[(*item)->tail->first_value].timestamp.sec - 1;
2012 }
2013 else
2014 range_end = ZBX_JAN_2038;
2015
2016 /* update cache if necessary */
2017 if (range_start >= range_end)
2018 return SUCCEED;
2019
2020 zbx_vector_history_record_create(&records);
2021 itemid = (*item)->itemid;
2022 value_type = (*item)->value_type;
2023
2024 UNLOCK_CACHE;
2025
2026 if (SUCCEED == (ret = vc_db_read_values_by_time(itemid, value_type, &records, range_start, range_end)))
2027 {
2028 zbx_vector_history_record_sort(&records,
2029 (zbx_compare_func_t)zbx_history_record_compare_asc_func);
2030 }
2031
2032 WRLOCK_CACHE;
2033
2034 if (SUCCEED != ret)
2035 goto out;
2036
2037 if (NULL == (*item = (zbx_vc_item_t *)zbx_hashset_search(&vc_cache->items, &itemid)))
2038 {
2039 zbx_vc_item_t new_item = {.itemid = itemid, .value_type = value_type};
2040
2041 if (NULL == (*item = (zbx_vc_item_t *)zbx_hashset_insert(&vc_cache->items, &new_item, sizeof(new_item))))
2042 goto out;
2043 }
2044
2045 /* when updating cache with time based request we can always reset status flags */
2046 /* flag even if the requested period contains no data */
2047 (*item)->status = 0;
2048
2049 if (0 < records.values_num)
2050 {
2051 if (SUCCEED != (ret = vch_item_add_values_at_tail(*item, records.values, records.values_num)))
2052 goto out;
2053 }
2054
2055 ret = records.values_num;
2056 vc_item_update_db_cached_from(*item, range_start);
2057 out:
2058 zbx_history_record_vector_destroy(&records, value_type);
2059
2060 return ret;
2061 }
2062
2063 /******************************************************************************
2064 * *
2065 * Function: vch_item_cache_values_by_time_and_count *
2066 * *
2067 * Purpose: cache the specified number of history data values for time period *
2068 * since timestamp *
2069 * *
2070 * Parameters: item - [IN] the item *
2071 * range_start - [IN] the interval start time *
2072 * count - [IN] the number of history values to retrieve *
2073 * ts - [IN] the target timestamp *
2074 * *
2075 * Return value: >=0 - the number of values read from database *
2076 * FAIL - an error occurred while trying to cache values *
2077 * *
2078 * Comments: This function checks if the requested number of values is cached *
2079 * and updates cache from database if necessary. *
2080 * *
2081 ******************************************************************************/
vch_item_cache_values_by_time_and_count(zbx_vc_item_t ** item,int range_start,int count,const zbx_timespec_t * ts)2082 static int vch_item_cache_values_by_time_and_count(zbx_vc_item_t **item, int range_start, int count,
2083 const zbx_timespec_t *ts)
2084 {
2085 int ret = SUCCEED, cached_records = 0, range_end, records_offset;
2086 zbx_vector_history_record_t records;
2087 zbx_uint64_t itemid;
2088 unsigned char value_type;
2089
2090 if (ZBX_ITEM_STATUS_CACHED_ALL == (*item)->status)
2091 return SUCCEED;
2092
2093 /* check if the requested period is in the cached range */
2094 if (0 != (*item)->db_cached_from && range_start >= (*item)->db_cached_from)
2095 return SUCCEED;
2096
2097 /* find if the cache should be updated to cover the required count */
2098 if (NULL != (*item)->head)
2099 {
2100 zbx_vc_chunk_t *chunk;
2101 int index;
2102
2103 if (SUCCEED == vch_item_get_last_value(*item, ts, &chunk, &index))
2104 {
2105 cached_records = index - chunk->first_value + 1;
2106
2107 while (NULL != (chunk = chunk->prev) && cached_records < count)
2108 cached_records += chunk->last_value - chunk->first_value + 1;
2109 }
2110 }
2111
2112 /* update cache if necessary */
2113 if (cached_records >= count)
2114 return SUCCEED;
2115
2116 /* get the end timestamp to which (including) the values should be cached */
2117 if (NULL != (*item)->head)
2118 range_end = (*item)->tail->slots[(*item)->tail->first_value].timestamp.sec - 1;
2119 else
2120 range_end = ZBX_JAN_2038;
2121
2122 itemid = (*item)->itemid;
2123 value_type = (*item)->value_type;
2124 UNLOCK_CACHE;
2125
2126 zbx_vector_history_record_create(&records);
2127
2128 if (range_end > ts->sec)
2129 {
2130 ret = vc_db_read_values_by_time(itemid, value_type, &records, ts->sec + 1, range_end);
2131 range_end = ts->sec;
2132 }
2133
2134 records_offset = records.values_num;
2135
2136 if (SUCCEED == ret && SUCCEED == (ret = vc_db_read_values_by_time_and_count(itemid, value_type, &records,
2137 range_start, count - cached_records, range_end, ts)))
2138 {
2139 zbx_vector_history_record_sort(&records,
2140 (zbx_compare_func_t)zbx_history_record_compare_asc_func);
2141 }
2142
2143 WRLOCK_CACHE;
2144
2145 if (SUCCEED != ret)
2146 goto out;
2147
2148 if (NULL == (*item = (zbx_vc_item_t *)zbx_hashset_search(&vc_cache->items, &itemid)))
2149 {
2150 zbx_vc_item_t new_item = {.itemid = itemid, .value_type = value_type};
2151
2152 if (NULL == (*item = (zbx_vc_item_t *)zbx_hashset_insert(&vc_cache->items, &new_item, sizeof(new_item))))
2153 goto out;
2154 }
2155
2156 if (0 < records.values_num)
2157 ret = vch_item_add_values_at_tail(*item, records.values, records.values_num);
2158
2159 if (SUCCEED != ret)
2160 goto out;
2161
2162 ret = records.values_num;
2163
2164 if (0 == range_start && count - cached_records > records.values_num - records_offset)
2165 {
2166 (*item)->active_range = 0;
2167 (*item)->daily_range = 0;
2168 (*item)->status = ZBX_ITEM_STATUS_CACHED_ALL;
2169 }
2170
2171 if ((count <= records.values_num || 0 == range_start) && 0 != records.values_num)
2172 {
2173 vc_item_update_db_cached_from(*item,
2174 (*item)->tail->slots[(*item)->tail->first_value].timestamp.sec);
2175 }
2176 else if (0 != range_start)
2177 vc_item_update_db_cached_from(*item, range_start);
2178
2179 out:
2180 zbx_history_record_vector_destroy(&records, value_type);
2181
2182 return ret;
2183 }
2184
2185 /******************************************************************************
2186 * *
2187 * Function: vch_item_get_values_by_time *
2188 * *
2189 * Purpose: retrieves item history data from cache *
2190 * *
2191 * Parameters: item - [IN] the item *
2192 * values - [OUT] the item history data stored time/value *
2193 * pairs in undefined order *
2194 * seconds - [IN] the time period to retrieve data for *
2195 * ts - [IN] the requested period end timestamp *
2196 * *
2197 ******************************************************************************/
vch_item_get_values_by_time(const zbx_vc_item_t * item,zbx_vector_history_record_t * values,int seconds,const zbx_timespec_t * ts)2198 static void vch_item_get_values_by_time(const zbx_vc_item_t *item, zbx_vector_history_record_t *values, int seconds,
2199 const zbx_timespec_t *ts)
2200 {
2201 int index, now;
2202 zbx_timespec_t start = {ts->sec - seconds, ts->ns};
2203 zbx_vc_chunk_t *chunk;
2204
2205 /* Check if maximum request range is not set and all data are cached. */
2206 /* Because that indicates there was a count based request with unknown */
2207 /* range which might be greater than the current request range. */
2208 if (0 != item->active_range || ZBX_ITEM_STATUS_CACHED_ALL != item->status)
2209 {
2210 now = time(NULL);
2211 /* add another second to include nanosecond shifts */
2212 vc_cache_item_update(item->itemid, ZBX_VC_UPDATE_RANGE, seconds + now - ts->sec + 1, now);
2213 }
2214
2215 if (FAIL == vch_item_get_last_value(item, ts, &chunk, &index))
2216 {
2217 /* Cache does not contain records for the specified timeshift & seconds range. */
2218 /* Return empty vector with success. */
2219 return;
2220 }
2221
2222 /* fill the values vector with item history values until the start timestamp is reached */
2223 while (0 < zbx_timespec_compare(&chunk->slots[chunk->last_value].timestamp, &start))
2224 {
2225 while (index >= chunk->first_value && 0 < zbx_timespec_compare(&chunk->slots[index].timestamp, &start))
2226 vc_history_record_vector_append(values, item->value_type, &chunk->slots[index--]);
2227
2228 if (NULL == (chunk = chunk->prev))
2229 break;
2230
2231 index = chunk->last_value;
2232 }
2233 }
2234
2235 /******************************************************************************
2236 * *
2237 * Function: vch_item_get_values_by_time_and_count *
2238 * *
2239 * Purpose: retrieves item history data from cache *
2240 * *
2241 * Parameters: item - [IN] the item *
2242 * values - [OUT] the item history data stored time/value *
2243 * pairs in undefined order, optional *
2244 * If null then cache is updated if necessary, but no *
2245 * values are returned. Used to ensure that cache *
2246 * contains a value of the specified timestamp. *
2247 * seconds - [IN] the time period *
2248 * count - [IN] the number of history values to retrieve *
2249 * timestamp - [IN] the target timestamp *
2250 * *
2251 ******************************************************************************/
vch_item_get_values_by_time_and_count(zbx_vc_item_t * item,zbx_vector_history_record_t * values,int seconds,int count,const zbx_timespec_t * ts)2252 static void vch_item_get_values_by_time_and_count(zbx_vc_item_t *item, zbx_vector_history_record_t *values,
2253 int seconds, int count, const zbx_timespec_t *ts)
2254 {
2255 int index, now, range_timestamp;
2256 zbx_vc_chunk_t *chunk;
2257 zbx_timespec_t start;
2258
2259 /* set start timestamp of the requested time period */
2260 if (0 != seconds)
2261 {
2262 start.sec = ts->sec - seconds;
2263 start.ns = ts->ns;
2264 }
2265 else
2266 {
2267 start.sec = 0;
2268 start.ns = 0;
2269 }
2270
2271 if (FAIL == vch_item_get_last_value(item, ts, &chunk, &index))
2272 {
2273 /* return empty vector with success */
2274 goto out;
2275 }
2276
2277 /* fill the values vector with item history values until the <count> values are read */
2278 /* or no more values within specified time period */
2279 /* fill the values vector with item history values until the start timestamp is reached */
2280 while (0 < zbx_timespec_compare(&chunk->slots[chunk->last_value].timestamp, &start))
2281 {
2282 while (index >= chunk->first_value && 0 < zbx_timespec_compare(&chunk->slots[index].timestamp, &start))
2283 {
2284 vc_history_record_vector_append(values, item->value_type, &chunk->slots[index--]);
2285
2286 if (values->values_num == count)
2287 goto out;
2288 }
2289
2290 if (NULL == (chunk = chunk->prev))
2291 break;
2292
2293 index = chunk->last_value;
2294 }
2295 out:
2296 if (count > values->values_num)
2297 {
2298 if (0 == seconds)
2299 return;
2300
2301 /* not enough data in the requested period, set the range equal to the period plus */
2302 /* one second to include nanosecond shifts */
2303 range_timestamp = ts->sec - seconds;
2304 }
2305 else
2306 {
2307 /* the requested number of values was retrieved, set the range to the oldest value timestamp */
2308 range_timestamp = values->values[values->values_num - 1].timestamp.sec - 1;
2309 }
2310
2311 now = time(NULL);
2312 vc_cache_item_update(item->itemid, ZBX_VC_UPDATE_RANGE, now - range_timestamp, now);
2313 }
2314
2315 /******************************************************************************
2316 * *
2317 * Function: vch_item_get_value_range *
2318 * *
2319 * Purpose: get item values for the specified range *
2320 * *
2321 * Parameters: item - [IN] the item *
2322 * values - [OUT] the item history data stored time/value *
2323 * pairs in undefined order, optional *
2324 * If null then cache is updated if necessary, but no *
2325 * values are returned. Used to ensure that cache *
2326 * contains a value of the specified timestamp. *
2327 * seconds - [IN] the time period to retrieve data for *
2328 * count - [IN] the number of history values to retrieve *
2329 * ts - [IN] the target timestamp *
2330 * *
2331 * Return value: SUCCEED - the item history data was retrieved successfully *
2332 * FAIL - the item history data was not retrieved *
2333 * *
2334 * Comments: This function returns data from cache if necessary updating it *
2335 * from DB. If cache update was required and failed (not enough *
2336 * memory to cache DB values), then this function also fails. *
2337 * *
2338 * If <count> is set then value range is defined as <count> values *
2339 * before <timestamp>. Otherwise the range is defined as <seconds> *
2340 * seconds before <timestamp>. *
2341 * *
2342 ******************************************************************************/
vch_item_get_values(zbx_vc_item_t * item,zbx_vector_history_record_t * values,int seconds,int count,const zbx_timespec_t * ts)2343 static int vch_item_get_values(zbx_vc_item_t *item, zbx_vector_history_record_t *values, int seconds,
2344 int count, const zbx_timespec_t *ts)
2345 {
2346 int ret, records_read, hits, misses, range_start;
2347
2348 zbx_vector_history_record_clear(values);
2349
2350 if (0 == count)
2351 {
2352 if (0 > (range_start = ts->sec - seconds))
2353 range_start = 0;
2354
2355 if (FAIL == (ret = vch_item_cache_values_by_time(&item, range_start)))
2356 goto out;
2357
2358 records_read = ret;
2359
2360 vch_item_get_values_by_time(item, values, seconds, ts);
2361
2362 if (records_read > values->values_num)
2363 records_read = values->values_num;
2364 }
2365 else
2366 {
2367 range_start = (0 == seconds ? 0 : ts->sec - seconds);
2368
2369 if (FAIL == (ret = vch_item_cache_values_by_time_and_count(&item, range_start, count, ts)))
2370 goto out;
2371
2372 records_read = ret;
2373
2374 vch_item_get_values_by_time_and_count(item, values, seconds, count, ts);
2375
2376 if (records_read > values->values_num)
2377 records_read = values->values_num;
2378 }
2379
2380 hits = values->values_num - records_read;
2381 misses = records_read;
2382
2383 vc_cache_item_update(item->itemid, ZBX_VC_UPDATE_STATS, hits, misses);
2384
2385 ret = SUCCEED;
2386 out:
2387 return ret;
2388 }
2389
2390 /******************************************************************************
2391 * *
2392 * Function: vch_item_free_cache *
2393 * *
2394 * Purpose: frees resources allocated for item history data *
2395 * *
2396 * Parameters: item - [IN] the item *
2397 * *
2398 * Return value: the size of freed memory (bytes) *
2399 * *
2400 ******************************************************************************/
vch_item_free_cache(zbx_vc_item_t * item)2401 static size_t vch_item_free_cache(zbx_vc_item_t *item)
2402 {
2403 size_t freed = 0;
2404
2405 zbx_vc_chunk_t *chunk = item->tail;
2406
2407 while (NULL != chunk)
2408 {
2409 zbx_vc_chunk_t *next = chunk->next;
2410
2411 freed += vch_item_free_chunk(item, chunk);
2412 chunk = next;
2413 }
2414 item->values_total = 0;
2415 item->head = NULL;
2416 item->tail = NULL;
2417
2418 return freed;
2419 }
2420
2421 /******************************************************************************************************************
2422 * *
2423 * Public API *
2424 * *
2425 ******************************************************************************************************************/
2426
2427 /******************************************************************************
2428 * *
2429 * Function: zbx_vc_init *
2430 * *
2431 * Purpose: initializes value cache *
2432 * *
2433 ******************************************************************************/
zbx_vc_init(char ** error)2434 int zbx_vc_init(char **error)
2435 {
2436 zbx_uint64_t size_reserved;
2437 int ret = FAIL;
2438
2439 if (0 == CONFIG_VALUE_CACHE_SIZE)
2440 return SUCCEED;
2441
2442 zabbix_log(LOG_LEVEL_DEBUG, "In %s()", __func__);
2443
2444 if (SUCCEED != (ret = zbx_rwlock_create(&vc_lock, ZBX_RWLOCK_VALUECACHE, error)))
2445 goto out;
2446
2447 size_reserved = zbx_mem_required_size(1, "value cache size", "ValueCacheSize");
2448
2449 if (SUCCEED != zbx_mem_create(&vc_mem, CONFIG_VALUE_CACHE_SIZE, "value cache size", "ValueCacheSize", 1, error))
2450 goto out;
2451
2452 CONFIG_VALUE_CACHE_SIZE -= size_reserved;
2453
2454 vc_cache = (zbx_vc_cache_t *)__vc_mem_malloc_func(vc_cache, sizeof(zbx_vc_cache_t));
2455
2456 if (NULL == vc_cache)
2457 {
2458 *error = zbx_strdup(*error, "cannot allocate value cache header");
2459 goto out;
2460 }
2461 memset(vc_cache, 0, sizeof(zbx_vc_cache_t));
2462
2463 zbx_hashset_create_ext(&vc_cache->items, VC_ITEMS_INIT_SIZE,
2464 ZBX_DEFAULT_UINT64_HASH_FUNC, ZBX_DEFAULT_UINT64_COMPARE_FUNC, NULL,
2465 __vc_mem_malloc_func, __vc_mem_realloc_func, __vc_mem_free_func);
2466
2467 if (NULL == vc_cache->items.slots)
2468 {
2469 *error = zbx_strdup(*error, "cannot allocate value cache data storage");
2470 goto out;
2471 }
2472
2473 zbx_hashset_create_ext(&vc_cache->strpool, VC_STRPOOL_INIT_SIZE,
2474 vc_strpool_hash_func, vc_strpool_compare_func, NULL,
2475 __vc_mem_malloc_func, __vc_mem_realloc_func, __vc_mem_free_func);
2476
2477 if (NULL == vc_cache->strpool.slots)
2478 {
2479 *error = zbx_strdup(*error, "cannot allocate string pool for value cache data storage");
2480 goto out;
2481 }
2482
2483 /* the free space request should be 5% of cache size, but no more than 128KB */
2484 vc_cache->min_free_request = (CONFIG_VALUE_CACHE_SIZE / 100) * 5;
2485 if (vc_cache->min_free_request > 128 * ZBX_KIBIBYTE)
2486 vc_cache->min_free_request = 128 * ZBX_KIBIBYTE;
2487
2488 zbx_vector_vc_itemupdate_create(&vc_itemupdates);
2489 zbx_vector_vc_itemupdate_reserve(&vc_itemupdates, 256);
2490
2491 ret = SUCCEED;
2492 out:
2493 zbx_vc_disable();
2494
2495 zabbix_log(LOG_LEVEL_DEBUG, "End of %s()", __func__);
2496
2497 return ret;
2498 }
2499
2500 /******************************************************************************
2501 * *
2502 * Function: zbx_vc_destroy *
2503 * *
2504 * Purpose: destroys value cache *
2505 * *
2506 ******************************************************************************/
zbx_vc_destroy(void)2507 void zbx_vc_destroy(void)
2508 {
2509 zabbix_log(LOG_LEVEL_DEBUG, "In %s()", __func__);
2510
2511 if (NULL != vc_cache)
2512 {
2513 zbx_vector_vc_itemupdate_destroy(&vc_itemupdates);
2514
2515 zbx_rwlock_destroy(&vc_lock);
2516
2517 zbx_hashset_destroy(&vc_cache->items);
2518 zbx_hashset_destroy(&vc_cache->strpool);
2519
2520 __vc_mem_free_func(vc_cache);
2521 vc_cache = NULL;
2522 }
2523
2524 zabbix_log(LOG_LEVEL_DEBUG, "End of %s()", __func__);
2525 }
2526
2527 /******************************************************************************
2528 * *
2529 * Function: zbx_vc_reset *
2530 * *
2531 * Purpose: resets value cache *
2532 * *
2533 * Comments: All items and their historical data are removed, *
2534 * cache working mode, statistics reset. *
2535 * *
2536 ******************************************************************************/
zbx_vc_reset(void)2537 void zbx_vc_reset(void)
2538 {
2539 zabbix_log(LOG_LEVEL_DEBUG, "In %s()", __func__);
2540
2541 if (NULL != vc_cache)
2542 {
2543 zbx_vc_item_t *item;
2544 zbx_hashset_iter_t iter;
2545
2546 WRLOCK_CACHE;
2547
2548 zbx_hashset_iter_reset(&vc_cache->items, &iter);
2549 while (NULL != (item = (zbx_vc_item_t *)zbx_hashset_iter_next(&iter)))
2550 {
2551 vch_item_free_cache(item);
2552 zbx_hashset_iter_remove(&iter);
2553 }
2554
2555 vc_cache->hits = 0;
2556 vc_cache->misses = 0;
2557 vc_cache->min_free_request = 0;
2558 vc_cache->mode = ZBX_VC_MODE_NORMAL;
2559 vc_cache->mode_time = 0;
2560 vc_cache->last_warning_time = 0;
2561
2562 UNLOCK_CACHE;
2563 }
2564
2565 zabbix_log(LOG_LEVEL_DEBUG, "End of %s()", __func__);
2566 }
2567
2568 /******************************************************************************
2569 * *
2570 * Function: zbx_vc_add_values *
2571 * *
2572 * Purpose: adds item values to the history and value cache *
2573 * *
2574 * Parameters: history - [IN] item history values *
2575 * *
2576 * Return value: SUCCEED - the values were added successfully *
2577 * FAIL - otherwise *
2578 * *
2579 ******************************************************************************/
zbx_vc_add_values(zbx_vector_ptr_t * history)2580 int zbx_vc_add_values(zbx_vector_ptr_t *history)
2581 {
2582 zbx_vc_item_t *item;
2583 int i;
2584 ZBX_DC_HISTORY *h;
2585 time_t expire_timestamp;
2586
2587 if (FAIL == zbx_history_add_values(history))
2588 return FAIL;
2589
2590 if (ZBX_VC_DISABLED == vc_state)
2591 return SUCCEED;
2592
2593 expire_timestamp = time(NULL) - ZBX_VC_ITEM_EXPIRE_PERIOD;
2594
2595 WRLOCK_CACHE;
2596
2597 for (i = 0; i < history->values_num; i++)
2598 {
2599 h = (ZBX_DC_HISTORY *)history->values[i];
2600
2601 if (NULL != (item = (zbx_vc_item_t *)zbx_hashset_search(&vc_cache->items, &h->itemid)))
2602 {
2603 zbx_history_record_t record = {h->ts, h->value};
2604 zbx_vc_chunk_t *head = item->head;
2605
2606 /* If the new value type does not match the item's type in cache remove it, */
2607 /* so it's cached with the correct type from correct tables when accessed */
2608 /* next time. */
2609 /* Also remove item if the value adding failed. In this case we */
2610 /* won't have the latest data in cache - so the requests must go directly */
2611 /* to the database. */
2612 if (item->value_type != h->value_type || item->last_accessed < expire_timestamp ||
2613 FAIL == vch_item_add_value_at_head(item, &record))
2614 {
2615 vc_remove_item(item);
2616 continue;
2617 }
2618
2619 /* try to remove old (unused) chunks if a new chunk was added */
2620 if (head != item->head)
2621 vch_item_clean_cache(item);
2622
2623 }
2624 }
2625
2626 UNLOCK_CACHE;
2627
2628 return SUCCEED;
2629 }
2630
2631 /******************************************************************************
2632 * *
2633 * Function: zbx_vc_get_values *
2634 * *
2635 * Purpose: get item history data for the specified time period *
2636 * *
2637 * Parameters: itemid - [IN] the item id *
2638 * value_type - [IN] the item value type *
2639 * values - [OUT] the item history data stored time/value *
2640 * pairs in descending order *
2641 * seconds - [IN] the time period to retrieve data for *
2642 * count - [IN] the number of history values to retrieve *
2643 * ts - [IN] the period end timestamp *
2644 * *
2645 * Return value: SUCCEED - the item history data was retrieved successfully *
2646 * FAIL - the item history data was not retrieved *
2647 * *
2648 * Comments: If the data is not in cache, it's read from DB, so this function *
2649 * will always return the requested data, unless some error occurs. *
2650 * *
2651 * If <count> is set then value range is defined as <count> values *
2652 * before <timestamp>. Otherwise the range is defined as <seconds> *
2653 * seconds before <timestamp>. *
2654 * *
2655 ******************************************************************************/
zbx_vc_get_values(zbx_uint64_t itemid,int value_type,zbx_vector_history_record_t * values,int seconds,int count,const zbx_timespec_t * ts)2656 int zbx_vc_get_values(zbx_uint64_t itemid, int value_type, zbx_vector_history_record_t *values, int seconds,
2657 int count, const zbx_timespec_t *ts)
2658 {
2659 zbx_vc_item_t *item, new_item;
2660 int ret = FAIL, cache_used = 1;
2661
2662 zabbix_log(LOG_LEVEL_DEBUG, "In %s() itemid:" ZBX_FS_UI64 " value_type:%d seconds:%d count:%d sec:%d ns:%d",
2663 __func__, itemid, value_type, seconds, count, ts->sec, ts->ns);
2664
2665 RDLOCK_CACHE;
2666
2667 if (ZBX_VC_DISABLED == vc_state)
2668 goto out;
2669
2670 if (ZBX_VC_MODE_LOWMEM == vc_cache->mode)
2671 vc_warn_low_memory();
2672
2673 if (NULL == (item = (zbx_vc_item_t *)zbx_hashset_search(&vc_cache->items, &itemid)))
2674 {
2675 if (ZBX_VC_MODE_NORMAL != vc_cache->mode)
2676 goto out;
2677
2678 memset(&new_item, 0, sizeof(new_item));
2679 new_item.itemid = itemid;
2680 new_item.value_type = value_type;
2681 item = &new_item;
2682 }
2683 else if (item->value_type != value_type)
2684 goto out;
2685
2686 ret = vch_item_get_values(item, values, seconds, count, ts);
2687 out:
2688 if (FAIL == ret)
2689 {
2690 cache_used = 0;
2691
2692 UNLOCK_CACHE;
2693 ret = vc_db_get_values(itemid, value_type, values, seconds, count, ts);
2694 WRLOCK_CACHE;
2695
2696 if (ZBX_VC_DISABLED != vc_state)
2697 vc_remove_item_by_id(itemid);
2698
2699 if (SUCCEED == ret)
2700 vc_update_statistics(NULL, 0, values->values_num, time(NULL));
2701 }
2702
2703 UNLOCK_CACHE;
2704
2705 zabbix_log(LOG_LEVEL_DEBUG, "End of %s():%s count:%d cached:%d",
2706 __func__, zbx_result_string(ret), values->values_num, cache_used);
2707
2708 return ret;
2709 }
2710
2711 /******************************************************************************
2712 * *
2713 * Function: zbx_vc_get_value *
2714 * *
2715 * Purpose: get the last history value with a timestamp less or equal to the *
2716 * target timestamp *
2717 * *
2718 * Parameters: itemid - [IN] the item id *
2719 * value_type - [IN] the item value type *
2720 * ts - [IN] the target timestamp *
2721 * value - [OUT] the value found *
2722 * *
2723 * Return Value: SUCCEED - the item was retrieved *
2724 * FAIL - otherwise *
2725 * *
2726 * Comments: Depending on the value type this function might allocate memory *
2727 * to store value data. To free it use zbx_vc_history_value_clear() *
2728 * function. *
2729 * *
2730 ******************************************************************************/
zbx_vc_get_value(zbx_uint64_t itemid,int value_type,const zbx_timespec_t * ts,zbx_history_record_t * value)2731 int zbx_vc_get_value(zbx_uint64_t itemid, int value_type, const zbx_timespec_t *ts, zbx_history_record_t *value)
2732 {
2733 zbx_vector_history_record_t values;
2734 int ret = FAIL;
2735
2736 zbx_history_record_vector_create(&values);
2737
2738 if (SUCCEED != zbx_vc_get_values(itemid, value_type, &values, ts->sec, 1, ts) || 0 == values.values_num)
2739 goto out;
2740
2741 *value = values.values[0];
2742
2743 /* reset values vector size so the returned value is not cleared when destroying the vector */
2744 values.values_num = 0;
2745
2746 ret = SUCCEED;
2747 out:
2748 zbx_history_record_vector_destroy(&values, value_type);
2749
2750 return ret;
2751 }
2752
2753 /******************************************************************************
2754 * *
2755 * Function: zbx_vc_get_statistics *
2756 * *
2757 * Purpose: retrieves usage cache statistics *
2758 * *
2759 * Parameters: stats - [OUT] the cache usage statistics *
2760 * *
2761 * Return value: SUCCEED - the cache statistics were retrieved successfully *
2762 * FAIL - failed to retrieve cache statistics *
2763 * (cache was not initialized) *
2764 * *
2765 ******************************************************************************/
zbx_vc_get_statistics(zbx_vc_stats_t * stats)2766 int zbx_vc_get_statistics(zbx_vc_stats_t *stats)
2767 {
2768 if (ZBX_VC_DISABLED == vc_state)
2769 return FAIL;
2770
2771 RDLOCK_CACHE;
2772
2773 stats->hits = vc_cache->hits;
2774 stats->misses = vc_cache->misses;
2775 stats->mode = vc_cache->mode;
2776
2777 stats->total_size = vc_mem->total_size;
2778 stats->free_size = vc_mem->free_size;
2779
2780 UNLOCK_CACHE;
2781
2782 return SUCCEED;
2783 }
2784
2785 /******************************************************************************
2786 * *
2787 * Function: zbx_vc_enable *
2788 * *
2789 * Purpose: enables value caching for current process *
2790 * *
2791 ******************************************************************************/
zbx_vc_enable(void)2792 void zbx_vc_enable(void)
2793 {
2794 if (NULL != vc_cache)
2795 vc_state = ZBX_VC_ENABLED;
2796 }
2797
2798 /******************************************************************************
2799 * *
2800 * Function: zbx_vc_disable *
2801 * *
2802 * Purpose: disables value caching for current process *
2803 * *
2804 ******************************************************************************/
zbx_vc_disable(void)2805 void zbx_vc_disable(void)
2806 {
2807 vc_state = ZBX_VC_DISABLED;
2808 }
2809
2810 /******************************************************************************
2811 * *
2812 * Function: zbx_hc_get_diag_stats *
2813 * *
2814 * Purpose: get value cache diagnostic statistics *
2815 * *
2816 ******************************************************************************/
zbx_vc_get_diag_stats(zbx_uint64_t * items_num,zbx_uint64_t * values_num,int * mode)2817 void zbx_vc_get_diag_stats(zbx_uint64_t *items_num, zbx_uint64_t *values_num, int *mode)
2818 {
2819 zbx_hashset_iter_t iter;
2820 zbx_vc_item_t *item;
2821
2822 *values_num = 0;
2823
2824 if (ZBX_VC_DISABLED == vc_state)
2825 {
2826 *items_num = 0;
2827 *mode = -1;
2828 return;
2829 }
2830
2831 RDLOCK_CACHE;
2832
2833 *items_num = vc_cache->items.num_data;
2834 *mode = vc_cache->mode;
2835
2836 zbx_hashset_iter_reset(&vc_cache->items, &iter);
2837 while (NULL != (item = (zbx_vc_item_t *)zbx_hashset_iter_next(&iter)))
2838 *values_num += item->values_total;
2839
2840 UNLOCK_CACHE;
2841 }
2842
2843 /******************************************************************************
2844 * *
2845 * Function: zbx_hc_get_mem_stats *
2846 * *
2847 * Purpose: get value cache shared memory statistics *
2848 * *
2849 ******************************************************************************/
zbx_vc_get_mem_stats(zbx_mem_stats_t * mem)2850 void zbx_vc_get_mem_stats(zbx_mem_stats_t *mem)
2851 {
2852 if (ZBX_VC_DISABLED == vc_state)
2853 {
2854 memset(mem, 0, sizeof(zbx_mem_stats_t));
2855 return;
2856 }
2857
2858 RDLOCK_CACHE;
2859 zbx_mem_get_stats(vc_mem, mem);
2860 UNLOCK_CACHE;
2861 }
2862
2863 /******************************************************************************
2864 * *
2865 * Function: zbx_vc_get_item_stats *
2866 * *
2867 * Purpose: get statistics of cached items *
2868 * *
2869 ******************************************************************************/
zbx_vc_get_item_stats(zbx_vector_ptr_t * stats)2870 void zbx_vc_get_item_stats(zbx_vector_ptr_t *stats)
2871 {
2872 zbx_hashset_iter_t iter;
2873 zbx_vc_item_t *item;
2874 zbx_vc_item_stats_t *item_stats;
2875
2876 if (ZBX_VC_DISABLED == vc_state)
2877 return;
2878
2879 RDLOCK_CACHE;
2880
2881 zbx_vector_ptr_reserve(stats, vc_cache->items.num_data);
2882
2883 zbx_hashset_iter_reset(&vc_cache->items, &iter);
2884 while (NULL != (item = (zbx_vc_item_t *)zbx_hashset_iter_next(&iter)))
2885 {
2886 item_stats = (zbx_vc_item_stats_t *)zbx_malloc(NULL, sizeof(zbx_vc_item_stats_t));
2887 item_stats->itemid = item->itemid;
2888 item_stats->values_num = item->values_total;
2889 item_stats->hourly_num = item->last_hourly_num;
2890 zbx_vector_ptr_append(stats, item_stats);
2891 }
2892
2893 UNLOCK_CACHE;
2894 }
2895
2896 /******************************************************************************
2897 * *
2898 * Function: zbx_vc_flush_stats *
2899 * *
2900 * Purpose: flush locally cached statistics *
2901 * *
2902 ******************************************************************************/
zbx_vc_flush_stats(void)2903 void zbx_vc_flush_stats(void)
2904 {
2905 int i, now;
2906 zbx_vc_item_t *item = NULL;
2907 zbx_uint64_t itemid = 0;
2908
2909 if (ZBX_VC_DISABLED == vc_state || 0 == vc_itemupdates.values_num)
2910 return;
2911
2912 zbx_vector_vc_itemupdate_sort(&vc_itemupdates, ZBX_DEFAULT_UINT64_COMPARE_FUNC);
2913
2914 now = time(NULL);
2915
2916 WRLOCK_CACHE;
2917
2918 for (i = 0; i < vc_itemupdates.values_num; i++)
2919 {
2920 zbx_vc_item_update_t *update = &vc_itemupdates.values[i];
2921
2922 if (itemid != update->itemid)
2923 {
2924 itemid = update->itemid;
2925 item = (zbx_vc_item_t *)zbx_hashset_search(&vc_cache->items, &itemid);
2926 }
2927
2928 if (NULL == item)
2929 continue;
2930
2931 switch (update->type)
2932 {
2933 case ZBX_VC_UPDATE_RANGE:
2934 vch_item_update_range(item, update->data[ZBX_VC_UPDATE_RANGE_SECONDS],
2935 update->data[ZBX_VC_UPDATE_RANGE_NOW]);
2936 break;
2937 case ZBX_VC_UPDATE_STATS:
2938 vc_update_statistics(item, update->data[ZBX_VC_UPDATE_STATS_HITS],
2939 update->data[ZBX_VC_UPDATE_STATS_MISSES], now);
2940 break;
2941 }
2942 }
2943
2944 UNLOCK_CACHE;
2945
2946 zbx_vector_vc_itemupdate_clear(&vc_itemupdates);
2947 }
2948
2949 #ifdef HAVE_TESTS
2950 # include "../../../tests/libs/zbxdbcache/valuecache_test.c"
2951 #endif
2952