1 /***************************************************************************** 2 * RRDtool 1.2.30 Copyright by Tobi Oetiker, 1997-2009 3 ***************************************************************************** 4 * rrd_format.h RRD Database Format header 5 *****************************************************************************/ 6 7 #ifndef _RRD_FORMAT_H 8 #define _RRD_FORMAT_H 9 10 #include "rrd.h" 11 12 /***************************************************************************** 13 * put this in your /usr/lib/magic file (/etc/magic on HPUX) 14 * 15 * # rrd database format 16 * 0 string RRD\0 rrd file 17 * >5 string >\0 version '%s' 18 * 19 *****************************************************************************/ 20 21 #define RRD_COOKIE "RRD" 22 /* #define RRD_VERSION "0002" */ 23 /* changed because microsecond precision requires another field */ 24 #define RRD_VERSION "0003" 25 #define FLOAT_COOKIE 8.642135E130 26 27 #include "rrd_nan_inf.h" 28 29 typedef union unival { 30 unsigned long u_cnt; 31 rrd_value_t u_val; 32 } unival; 33 34 35 /**************************************************************************** 36 * The RRD Database Structure 37 * --------------------------- 38 * 39 * In oder to properly describe the database structure lets define a few 40 * new words: 41 * 42 * ds - Data Source (ds) providing input to the database. A Data Source (ds) 43 * can be a traffic counter, a temperature, the number of users logged 44 * into a system. The rrd database format can handle the input of 45 * several Data Sources (ds) in a singe database. 46 * 47 * dst - Data Source Type (dst). The Data Source Type (dst) defines the rules 48 * applied to Build Primary Data Points from the input provided by the 49 * data sources (ds). 50 * 51 * pdp - Primary Data Point (pdp). After the database has accepted the 52 * input from the data sources (ds). It starts building Primary 53 * Data Points (pdp) from the data. Primary Data Points (pdp) 54 * are evenly spaced along the time axis (pdp_step). The values 55 * of the Primary Data Points are calculated from the values of 56 * the data source (ds) and the exact time these values were 57 * provided by the data source (ds). 58 * 59 * pdp_st - PDP Start (pdp_st). The moments (pdp_st) in time where 60 * these steps occur are defined by the moments where the 61 * number of seconds since 1970-jan-1 modulo pdp_step equals 62 * zero (pdp_st). 63 * 64 * cf - Consolidation Function (cf). An arbitrary Consolidation Function (cf) 65 * (averaging, min, max) is applied to the primary data points (pdp) to 66 * calculate the consolidated data point. 67 * 68 * cdp - Consolidated Data Point (cdp) is the long term storage format for data 69 * in the rrd database. Consolidated Data Points represent one or 70 * several primary data points collected along the time axis. The 71 * Consolidated Data Points (cdp) are stored in Round Robin Archives 72 * (rra). 73 * 74 * rra - Round Robin Archive (rra). This is the place where the 75 * consolidated data points (cdp) get stored. The data is 76 * organized in rows (row) and columns (col). The Round Robin 77 * Archive got its name from the method data is stored in 78 * there. An RRD database can contain several Round Robin 79 * Archives. Each Round Robin Archive can have a different row 80 * spacing along the time axis (pdp_cnt) and a different 81 * consolidation function (cf) used to build its consolidated 82 * data points (cdp). 83 * 84 * rra_st - RRA Start (rra_st). The moments (rra_st) in time where 85 * Consolidated Data Points (cdp) are added to an rra are 86 * defined by the moments where the number of seconds since 87 * 1970-jan-1 modulo pdp_cnt*pdp_step equals zero (rra_st). 88 * 89 * row - Row (row). A row represent all consolidated data points (cdp) 90 * in a round robin archive who are of the same age. 91 * 92 * col - Column (col). A column (col) represent all consolidated 93 * data points (cdp) in a round robin archive (rra) who 94 * originated from the same data source (ds). 95 * 96 */ 97 98 /**************************************************************************** 99 * POS 1: stat_head_t static header of the database 100 ****************************************************************************/ 101 102 typedef struct stat_head_t { 103 104 /* Data Base Identification Section ***/ 105 char cookie[4]; /* RRD */ 106 char version[5]; /* version of the format */ 107 double float_cookie; /* is it the correct double 108 * representation ? */ 109 110 /* Data Base Structure Definition *****/ 111 unsigned long ds_cnt; /* how many different ds provide 112 * input to the rrd */ 113 unsigned long rra_cnt; /* how many rras will be maintained 114 * in the rrd */ 115 unsigned long pdp_step; /* pdp interval in seconds */ 116 117 unival par[10]; /* global parameters ... unused 118 at the moment */ 119 } stat_head_t; 120 121 122 /**************************************************************************** 123 * POS 2: ds_def_t (* ds_cnt) Data Source definitions 124 ****************************************************************************/ 125 126 enum dst_en { DST_COUNTER=0, /* data source types available */ 127 DST_ABSOLUTE, 128 DST_GAUGE, 129 DST_DERIVE, 130 DST_CDEF}; 131 132 enum ds_param_en { DS_mrhb_cnt=0, /* minimum required heartbeat. A 133 * data source must provide input at 134 * least every ds_mrhb seconds, 135 * otherwise it is regarded dead and 136 * will be set to UNKNOWN */ 137 DS_min_val, /* the processed input of a ds must */ 138 DS_max_val, /* be between max_val and min_val 139 * both can be set to UNKNOWN if you 140 * do not care. Data outside the limits 141 * set to UNKNOWN */ 142 DS_cdef = DS_mrhb_cnt}; /* pointer to encoded rpn 143 * expression only applies to DST_CDEF */ 144 145 /* The magic number here is one less than DS_NAM_SIZE */ 146 #define DS_NAM_FMT "%19[a-zA-Z0-9_-]" 147 #define DS_NAM_SIZE 20 148 149 #define DST_FMT "%19[A-Z]" 150 #define DST_SIZE 20 151 152 typedef struct ds_def_t { 153 char ds_nam[DS_NAM_SIZE]; /* Name of the data source (null terminated)*/ 154 char dst[DST_SIZE]; /* Type of data source (null terminated)*/ 155 unival par[10]; /* index of this array see ds_param_en */ 156 } ds_def_t; 157 158 /**************************************************************************** 159 * POS 3: rra_def_t ( * rra_cnt) one for each store to be maintained 160 ****************************************************************************/ 161 enum cf_en { CF_AVERAGE=0, /* data consolidation functions */ 162 CF_MINIMUM, 163 CF_MAXIMUM, 164 CF_LAST, 165 CF_HWPREDICT, 166 /* An array of predictions using the seasonal 167 * Holt-Winters algorithm. Requires an RRA of type 168 * CF_SEASONAL for this data source. */ 169 CF_SEASONAL, 170 /* An array of seasonal effects. Requires an RRA of 171 * type CF_HWPREDICT for this data source. */ 172 CF_DEVPREDICT, 173 /* An array of deviation predictions based upon 174 * smoothed seasonal deviations. Requires an RRA of 175 * type CF_DEVSEASONAL for this data source. */ 176 CF_DEVSEASONAL, 177 /* An array of smoothed seasonal deviations. Requires 178 * an RRA of type CF_HWPREDICT for this data source. 179 * */ 180 CF_FAILURES}; 181 /* A binary array of failure indicators: 1 indicates 182 * that the number of violations in the prescribed 183 * window exceeded the prescribed threshold. */ 184 185 #define MAX_RRA_PAR_EN 10 186 enum rra_par_en { RRA_cdp_xff_val=0, /* what part of the consolidated 187 * datapoint must be known, to produce a 188 * valid entry in the rra */ 189 RRA_hw_alpha, 190 /* exponential smoothing parameter for the intercept in 191 * the Holt-Winters prediction algorithm. */ 192 RRA_hw_beta, 193 /* exponential smoothing parameter for the slope in 194 * the Holt-Winters prediction algorithm. */ 195 RRA_dependent_rra_idx, 196 /* For CF_HWPREDICT: index of the RRA with the seasonal 197 * effects of the Holt-Winters algorithm (of type 198 * CF_SEASONAL). 199 * For CF_DEVPREDICT: index of the RRA with the seasonal 200 * deviation predictions (of type CF_DEVSEASONAL). 201 * For CF_SEASONAL: index of the RRA with the Holt-Winters 202 * intercept and slope coefficient (of type CF_HWPREDICT). 203 * For CF_DEVSEASONAL: index of the RRA with the 204 * Holt-Winters prediction (of type CF_HWPREDICT). 205 * For CF_FAILURES: index of the CF_DEVSEASONAL array. 206 * */ 207 RRA_seasonal_smooth_idx, 208 /* For CF_SEASONAL and CF_DEVSEASONAL: 209 * an integer between 0 and row_count - 1 which 210 * is index in the seasonal cycle for applying 211 * the period smoother. */ 212 RRA_failure_threshold, 213 /* For CF_FAILURES, number of violations within the last 214 * window required to mark a failure. */ 215 RRA_seasonal_gamma = RRA_hw_alpha, 216 /* exponential smoothing parameter for seasonal effects. 217 * */ 218 RRA_delta_pos = RRA_hw_alpha, 219 RRA_delta_neg = RRA_hw_beta, 220 /* confidence bound scaling parameters for the 221 * the FAILURES RRA. */ 222 RRA_window_len = RRA_seasonal_smooth_idx}; 223 /* For CF_FAILURES, the length of the window for measuring 224 * failures. */ 225 226 #define CF_NAM_FMT "%19[A-Z]" 227 #define CF_NAM_SIZE 20 228 229 typedef struct rra_def_t { 230 char cf_nam[CF_NAM_SIZE];/* consolidation function (null term) */ 231 unsigned long row_cnt; /* number of entries in the store */ 232 unsigned long pdp_cnt; /* how many primary data points are 233 * required for a consolidated data 234 * point?*/ 235 unival par[MAX_RRA_PAR_EN]; /* index see rra_param_en */ 236 237 } rra_def_t; 238 239 240 /**************************************************************************** 241 **************************************************************************** 242 **************************************************************************** 243 * LIVE PART OF THE HEADER. THIS WILL BE WRITTEN ON EVERY UPDATE * 244 **************************************************************************** 245 **************************************************************************** 246 ****************************************************************************/ 247 /**************************************************************************** 248 * POS 4: live_head_t 249 ****************************************************************************/ 250 251 typedef struct live_head_t { 252 time_t last_up; /* when was rrd last updated */ 253 long last_up_usec; /* micro seconds part of the 254 update timestamp. Always >= 0 */ 255 } live_head_t; 256 257 258 /**************************************************************************** 259 * POS 5: pdp_prep_t (* ds_cnt) here we prepare the pdps 260 ****************************************************************************/ 261 #define LAST_DS_LEN 30 /* DO NOT CHANGE THIS ... */ 262 263 enum pdp_par_en { PDP_unkn_sec_cnt=0, /* how many seconds of the current 264 * pdp value is unknown data? */ 265 266 PDP_val}; /* current value of the pdp. 267 this depends on dst */ 268 269 typedef struct pdp_prep_t{ 270 char last_ds[LAST_DS_LEN]; /* the last reading from the data 271 * source. this is stored in ASCII 272 * to cater for very large counters 273 * we might encounter in connection 274 * with SNMP. */ 275 unival scratch[10]; /* contents according to pdp_par_en */ 276 } pdp_prep_t; 277 278 /* data is passed from pdp to cdp when seconds since epoch modulo pdp_step == 0 279 obviously the updates do not occur at these times only. Especially does the 280 format allow for updates to occur at different times for each data source. 281 The rules which makes this work is as follows: 282 283 * DS updates may only occur at ever increasing points in time 284 * When any DS update arrives after a cdp update time, the *previous* 285 update cycle gets executed. All pdps are transfered to cdps and the 286 cdps feed the rras where necessary. Only then the new DS value 287 is loaded into the PDP. */ 288 289 290 /**************************************************************************** 291 * POS 6: cdp_prep_t (* rra_cnt * ds_cnt ) data prep area for cdp values 292 ****************************************************************************/ 293 #define MAX_CDP_PAR_EN 10 294 #define MAX_CDP_FAILURES_IDX 8 295 /* max CDP scratch entries avail to record violations for a FAILURES RRA */ 296 #define MAX_FAILURES_WINDOW_LEN 28 297 enum cdp_par_en { CDP_val=0, 298 /* the base_interval is always an 299 * average */ 300 CDP_unkn_pdp_cnt, 301 /* how many unknown pdp were 302 * integrated. This and the cdp_xff 303 * will decide if this is going to 304 * be a UNKNOWN or a valid value */ 305 CDP_hw_intercept, 306 /* Current intercept coefficient for the Holt-Winters 307 * prediction algorithm. */ 308 CDP_hw_last_intercept, 309 /* Last iteration intercept coefficient for the Holt-Winters 310 * prediction algorihtm. */ 311 CDP_hw_slope, 312 /* Current slope coefficient for the Holt-Winters 313 * prediction algorithm. */ 314 CDP_hw_last_slope, 315 /* Last iteration slope coeffient. */ 316 CDP_null_count, 317 /* Number of sequential Unknown (DNAN) values + 1 preceding 318 * the current prediction. 319 * */ 320 CDP_last_null_count, 321 /* Last iteration count of Unknown (DNAN) values. */ 322 CDP_primary_val = 8, 323 /* optimization for bulk updates: the value of the first CDP 324 * value to be written in the bulk update. */ 325 CDP_secondary_val = 9, 326 /* optimization for bulk updates: the value of subsequent 327 * CDP values to be written in the bulk update. */ 328 CDP_hw_seasonal = CDP_hw_intercept, 329 /* Current seasonal coefficient for the Holt-Winters 330 * prediction algorithm. This is stored in CDP prep to avoid 331 * redundant seek operations. */ 332 CDP_hw_last_seasonal = CDP_hw_last_intercept, 333 /* Last iteration seasonal coeffient. */ 334 CDP_seasonal_deviation = CDP_hw_intercept, 335 CDP_last_seasonal_deviation = CDP_hw_last_intercept, 336 CDP_init_seasonal = CDP_null_count}; 337 /* init_seasonal is a flag which when > 0, forces smoothing updates 338 * to occur when rra_ptr.cur_row == 0 */ 339 340 typedef struct cdp_prep_t{ 341 unival scratch[MAX_CDP_PAR_EN]; 342 /* contents according to cdp_par_en * 343 * init state should be NAN */ 344 345 } cdp_prep_t; 346 347 /**************************************************************************** 348 * POS 7: rra_ptr_t (* rra_cnt) pointers to the current row in each rra 349 ****************************************************************************/ 350 351 typedef struct rra_ptr_t { 352 unsigned long cur_row; /* current row in the rra*/ 353 } rra_ptr_t; 354 355 356 /**************************************************************************** 357 **************************************************************************** 358 * One single struct to hold all the others. For convenience. 359 **************************************************************************** 360 ****************************************************************************/ 361 typedef struct rrd_t { 362 stat_head_t *stat_head; /* the static header */ 363 ds_def_t *ds_def; /* list of data source definitions */ 364 rra_def_t *rra_def; /* list of round robin archive def */ 365 live_head_t *live_head; 366 pdp_prep_t *pdp_prep; /* pdp data prep area */ 367 cdp_prep_t *cdp_prep; /* cdp prep area */ 368 rra_ptr_t *rra_ptr; /* list of rra pointers */ 369 rrd_value_t *rrd_value; /* list of rrd values */ 370 } rrd_t; 371 372 /**************************************************************************** 373 **************************************************************************** 374 * AFTER the header section we have the DATA STORAGE AREA it is made up from 375 * Consolidated Data Points organized in Round Robin Archives. 376 **************************************************************************** 377 **************************************************************************** 378 379 *RRA 0 380 (0,0) .................... ( ds_cnt -1 , 0) 381 . 382 . 383 . 384 (0, row_cnt -1) ... (ds_cnt -1, row_cnt -1) 385 386 *RRA 1 387 *RRA 2 388 389 *RRA rra_cnt -1 390 391 ****************************************************************************/ 392 393 394 #endif 395 396 397 398 399