1<?php 2 3/** 4 * Observium 5 * 6 * This file is part of Observium. 7 * 8 * @package observium 9 * @subpackage poller 10 * @author Adam Armstrong <adama@observium.org> 11 * @copyright (C) 2006-2013 Adam Armstrong, (C) 2013-2019 Observium Limited 12 * 13 */ 14 15unset($cache['devices']['uptime'][$device['device_id']]); 16 17$poll_device = array(); 18 19$include_order = 'default'; // Default MIBs first (not sure, need more use cases) 20$include_dir = "includes/polling/system"; 21include("includes/include-dir-mib.inc.php"); 22 23// 5. Always keep SNMPv2-MIB::sysUpTime.0 as last point for uptime 24$uptimes = array('use' => 'sysUpTime', 25 'sysUpTime' => $poll_device['sysUpTime']); 26 27// Find MIB-specific SNMP data via OID fetch: sysDescr, sysLocation, sysContact, sysName, sysUpTime 28$system_metatypes = array('sysDescr', 'sysLocation', 'sysContact', 'sysName', 'sysUpTime', 'reboot'); // 'snmpEngineID'); 29foreach ($system_metatypes as $metatype) 30{ 31 32 $param = ($metatype == 'sysUpTime') ? 'uptime' : strtolower($metatype); // For sysUptime use simple param name 33 foreach (get_device_mibs_permitted($device) as $mib) // Check every MIB supported by the device, in order 34 { 35 if (isset($config['mibs'][$mib][$param])) 36 { 37 foreach ($config['mibs'][$mib][$param] as $entry) 38 { 39 // For ability override metatype by vendor mib definition use 'force' boolean param 40 if (!isset($poll_device[$metatype]) || // Poll if metatype not already set 41 $metatype == 'sysUpTime' || $metatype == 'reboot' || // Or uptime/reboot (always polled) 42 (isset($entry['override']) && $entry['override'])) // Or forced override (see ekinops EKINOPS-MGNT2-MIB mib definition 43 { 44 if (isset($entry['oid_num'])) // Use numeric OID if set, otherwise fetch text based string 45 { 46 $value = trim(snmp_hexstring(snmp_get_oid($device, $entry['oid_num']))); 47 } 48 elseif (isset($entry['oid_next'])) 49 { 50 // If Oid passed without index part use snmpgetnext (see FCMGMT-MIB definitions) 51 $value = trim(snmp_hexstring(snmp_getnext_oid($device, $entry['oid_next'], $mib))); 52 } else { 53 $value = trim(snmp_hexstring(snmp_get_oid($device, $entry['oid'], $mib))); 54 } 55 56 if ($GLOBALS['snmp_status'] && $value != '') 57 { 58 $polled = round($GLOBALS['exec_status']['endtime']); 59 60 // Field found (no SNMP error), perform optional transformations. 61 $value = string_transform($value, $entry['transformations']); 62 63 // Detect uptime with MIB defined oids, see below uptimes 2. 64 if ($metatype == 'sysUpTime' || $metatype == 'reboot') 65 { 66 // Previous uptime from standard sysUpTime or other MIB::oid 67 $uptime_previous = isset($poll_device['device_uptime']) ? $poll_device['device_uptime'] : $poll_device['sysUpTime']; 68 if ($metatype == 'reboot' && is_numeric($value)) 69 { 70 // Last reboot is same uptime, but as diff with current (polled) time (see INFINERA-ENTITY-CHASSIS-MIB) 71 $value = $polled - $value; 72 } 73 // Detect if new sysUpTime value more than the previous 74 if (is_numeric($value) && $value >= $uptime_previous) 75 { 76 $poll_device['device_uptime'] = $value; 77 $uptimes['message'] = isset($entry['oid_num']) ? $entry['oid_num'] : $mib . '::' . $entry['oid']; 78 $uptimes['message'] = 'Using device MIB poller '.$metatype.': ' . $uptimes['message']; 79 80 print_debug("Added System Uptime from SNMP definition walk: 'device_uptime' = '$value'"); 81 } 82 // Continue for other possible sysUpTime and use maximum value 83 continue; 84 } 85 86 $poll_device[$metatype] = $value; 87 print_debug("Added System param from SNMP definition walk: '$metatype' = '$value'"); 88 89 // Exit both foreach loops and move on to the next field. 90 break 2; 91 } 92 } 93 } 94 } 95 } 96 97} 98$poll_device['sysName_original'] = $poll_device['sysName']; // Store original sysName for devices who store hardware in this Oid 99$poll_device['sysName'] = strtolower($poll_device['sysName']); 100print_debug_vars($poll_device); 101 102// If polled time not set by MIB include, set to unixtime 103if (!isset($polled)) 104{ 105 $polled = time(); 106} 107 108// Uptime data and reboot triggers 109// NOTES. http://net-snmp.sourceforge.net/docs/FAQ.html#The_system_uptime__sysUpTime__returned_is_wrong_ 110// According to it, sysUptime reports time since last snmpd restart, while 111// hrSystemUptime reports time since last system reboot. 112// And prefer snmpEngineTime over hrSystemUptime and sysUptime, since they limited with 497 days 113 114// 5. As last point used sysUptime (see above) 115 116// 1. Unix-agent uptime is highest priority, since mostly accurate 117if (isset($agent_data['uptime'])) 118{ 119 list($agent_data['uptime']) = explode(' ', $agent_data['uptime']); 120 $uptimes['unix-agent'] = round($agent_data['uptime']); 121} 122 123if (is_numeric($agent_data['uptime']) && $agent_data['uptime'] > 0) 124{ 125 $uptimes['use'] = 'unix-agent'; 126 $uptimes['message'] = 'Using UNIX Agent Uptime'; 127} 128// 2. Uptime from os specific OID, see in includes/polling/system MIB specific 129else if (isset($poll_device['device_uptime']) && is_numeric($poll_device['device_uptime']) && $poll_device['device_uptime'] > 0) 130{ 131 // Get uptime by some custom way in device os poller, see example in wowza-engine os poller 132 $uptimes['device_uptime'] = round($poll_device['device_uptime']); 133 $uptimes['use'] = 'device_uptime'; 134 $uptimes['message'] = ($uptimes['message']) ? $uptimes['message'] : 'Using device MIB poller Uptime'; 135} else { 136 // 3. Uptime from hrSystemUptime (only in snmp v2c/v3) 137 // NOTE, about windows uptime, 138 // sysUpTime resets when SNMP service restarted, but hrSystemUptime resets at 49.7 days (always), 139 // Now we use LanMgr-Mib-II-MIB::comStatStart.0 as reboot time instead 140 if ($device['os'] != 'windows' && 141 $device['snmp_version'] != 'v1' && is_device_mib($device, 'HOST-RESOURCES-MIB')) 142 { 143 // HOST-RESOURCES-MIB::hrSystemUptime.0 = Wrong Type (should be Timeticks): 1632295600 144 // HOST-RESOURCES-MIB::hrSystemUptime.0 = Timeticks: (63050465) 7 days, 7:08:24.65 145 $hrSystemUptime = snmp_get_oid($device, 'hrSystemUptime.0', 'HOST-RESOURCES-MIB'); 146 $uptimes['hrSystemUptime'] = timeticks_to_sec($hrSystemUptime); 147 148 if (is_numeric($uptimes['hrSystemUptime']) && $uptimes['hrSystemUptime'] > 0) 149 { 150 // hrSystemUptime always prefer if not zero 151 $uptimes['use'] = 'hrSystemUptime'; 152 } 153 } 154 155 // 4. Uptime from snmpEngineTime (only in snmp v2c/v3) 156 if ($device['snmp_version'] != 'v1' && is_device_mib($device, 'SNMP-FRAMEWORK-MIB')) 157 { 158 $snmpEngineTime = snmp_get_oid($device, 'snmpEngineTime.0', 'SNMP-FRAMEWORK-MIB'); 159 160 if (is_numeric($snmpEngineTime) && $snmpEngineTime > 0) 161 { 162 if ($device['os'] == 'aos' && strlen($snmpEngineTime) > 8) 163 { 164 // Some Alcatel have bug with snmpEngineTime 165 // See: http://jira.observium.org/browse/OBSERVIUM-763 166 $snmpEngineTime = 0; 167 } 168 else if ($device['os'] == 'ironware') 169 { 170 // Check if version correct like "07.4.00fT7f3" 171 $ironware_version = explode('.', $device['version']); 172 if (count($ironware_version) > 2 && $ironware_version[0] > 0 && version_compare($device['version'], '5.1.0') === -1) 173 { 174 // IronWare before Release 05.1.00b have bug (firmware returning snmpEngineTime * 10) 175 // See: http://jira.observium.org/browse/OBSERVIUM-1199 176 $snmpEngineTime = $snmpEngineTime / 10; 177 } 178 } 179 $uptimes['snmpEngineTime'] = intval($snmpEngineTime); 180 181 if ($uptimes['use'] == 'hrSystemUptime') 182 { 183 // Prefer snmpEngineTime only if more than hrSystemUptime 184 if ($uptimes['snmpEngineTime'] > $uptimes['hrSystemUptime']) { $uptimes['use'] = 'snmpEngineTime'; } 185 } 186 else if ($uptimes['snmpEngineTime'] >= $uptimes['sysUpTime']) 187 { 188 // in other cases prefer if more than sysUpTime 189 $uptimes['use'] = 'snmpEngineTime'; 190 } 191 } 192 } 193 194} 195 196$uptimes['uptime'] = $uptimes[$uptimes['use']]; // Get actual uptime based on use flag 197$uptimes['formatted'] = format_uptime($uptimes['uptime']); // Human readable uptime 198if (!isset($uptimes['message'])) { $uptimes['message'] = 'Using SNMP Agent '.$uptimes['use']; } 199 200$uptime = $uptimes['uptime']; 201print_debug($uptimes['message']." ($uptime sec. => ".$uptimes['formatted'].')'); 202 203if (is_numeric($uptime) && $uptime > 0) // it really is very impossible case for $uptime equals to zero 204{ 205 $uptimes['previous'] = $device['uptime']; // Uptime from previous device poll 206 $uptimes['diff'] = $uptimes['previous'] - $uptime; // Difference between previous and current uptimes 207 208 // Calculate current last rebooted time 209 $last_rebooted = $polled - $uptime; 210 // Previous reboot unixtime 211 $uptimes['last_rebooted'] = $device['last_rebooted']; 212 if (empty($uptimes['last_rebooted']) || // 0 or '' 213 abs($device['last_rebooted'] - $last_rebooted) > 1200) // Fix when uptime updated by other Oid 214 { 215 // Set last_rebooted for all devices who not have it already 216 $uptimes['last_rebooted'] = $last_rebooted; 217 $update_array['last_rebooted'] = $last_rebooted; 218 } 219 220 // Notify only if current uptime less than one week (eg if changed from sysUpTime to snmpEngineTime) 221 $rebooted = 0; 222 if ($uptime < 604800) 223 { 224 if ($uptimes['diff'] > 60) 225 { 226 // If difference betwen old uptime ($device['uptime']) and new ($uptime) 227 // greater than 60 sec, than device truly rebooted 228 $rebooted = 1; 229 } 230 else if ($uptimes['previous'] < 300 && abs($uptimes['diff']) < 280) 231 { 232 // This is rare, boundary case, when device rebooted multiple times betwen polling runs 233 $rebooted = 1; 234 } 235 236 // Fix reboot flag with some borderline states (roll over counters) 237 if ($rebooted) 238 { 239 switch($uptimes['use']) 240 { 241 case 'hrSystemUptime': 242 case 'sysUpTime': 243 $uptimes_max = array(42949673); // 497 days 2 hours 27 minutes 53 seconds, counter 2^32 (4294967296) divided by 100 244 break; 245 case 'snmpEngineTime': 246 $uptimes_max = array(2147483647); // Average 68.05 years, counter is 2^32 (4294967296) divided by 2 247 break; 248 default: 249 // By default uptime limited only by PHP max values 250 // Usually int(2147483647) in 32 bit systems and int(9223372036854775807) in 64 bit systems 251 $uptimes_max = array(PHP_INT_MAX); 252 } 253 if (isset($config['os'][$device['os']]['uptime_max'][$uptimes['use']])) 254 { 255 // Add rollover counter time from definitions 256 $uptimes_max = array_merge($uptimes_max, (array)$config['os'][$device['os']]['uptime_max'][$uptimes['use']]); 257 } 258 // Exclude uptime counter rollover 259 /** 260 * Examples with APC PDU (rollover max sysUpTime is 49 days, 17h 2m 47s): 261 * 1. rebooted uptime previous: 49 days, 16h 52m 18s 262 * less than max: 10m 29s 263 * 2. rebooted uptime previous: 49 days, 16h 54m 50s 264 * less than max: 7m 57s 265 */ 266 foreach ($uptimes_max as $max) 267 { 268 // Exclude 660 sec (11 min) from maximal 269 if ($uptimes['previous'] > ($max - 660) && $uptimes['previous'] <= $max) 270 { 271 $uptimes['max'] = $max; 272 $rebooted = 0; 273 break; 274 } 275 } 276 } 277 278 if ($rebooted) 279 { 280 $update_array['last_rebooted'] = $polled - $uptime; // Update last reboot unixtime 281 //$reboot_diff = $update_array['last_rebooted'] - $uptimes['last_rebooted']; // Calculate time between 2 reboots 282 log_event('Device rebooted: after '.format_uptime($polled - $uptimes['last_rebooted']) . ' (Uptime: '.$uptimes['formatted'].', previous: '.format_uptime($uptimes['previous']).', used: '.$uptimes['use'] . ')', $device, 'device', $device['device_id'], 4); 283 $uptimes['last_rebooted'] = $update_array['last_rebooted']; // store new reboot unixtime 284 } 285 } 286 $uptimes['rebooted'] = $rebooted; 287 288 rrdtool_update_ng($device, 'uptime', array('uptime' => $uptime)); 289 290 $graphs['uptime'] = TRUE; 291 292 print_cli_data('Uptime', $uptimes['formatted']); 293 print_cli_data('Last reboot', format_unixtime($uptimes['last_rebooted'])); 294 295 $update_array['uptime'] = $uptime; 296 $cache['devices']['uptime'][$device['device_id']]['uptime'] = $uptime; 297 $cache['devices']['uptime'][$device['device_id']]['sysUpTime'] = $uptimes['sysUpTime']; // Required for ports (ifLastChange) 298 $cache['devices']['uptime'][$device['device_id']]['polled'] = $polled; 299} else { 300 print_warning('Device does not have any uptime counter or uptime equals zero.'); 301} 302print_debug_vars($uptimes, 1); 303 304// Load Average 305 306foreach (get_device_mibs_permitted($device) as $mib) // Check every MIB supported by the device 307{ 308 if (isset($config['mibs'][$mib]['la'])) 309 { 310 $la_def = $config['mibs'][$mib]['la']; 311 312 // FIXME. Filter hack, need more common way! 313 // See Cisco CISCO-PROCESS-MIB 314 foreach ($la_def['filter'] as $param => $entries) 315 { 316 if (!in_array($device[$param], (array)$entries)) { continue; } 317 } 318 319 $la_oids = array(); 320 if (isset($la_def['type']) && $la_def['type'] == 'table') 321 { 322 // First element from table walk, currently only for Cisco IOS-XE 323 $la_oids = ['1min' => $la_def['oid_1min'], '5min' => $la_def['oid_5min'], '15min' => $la_def['oid_15min']]; 324 $la = snmpwalk_cache_oid($device, $la_def['oid_1min'], array(), $mib); 325 $la = snmpwalk_cache_oid($device, $la_def['oid_5min'], $la, $mib); 326 $la = snmpwalk_cache_oid($device, $la_def['oid_15min'], $la, $mib); 327 $la = array_shift($la); 328 } else { 329 // Single oid 330 foreach (array('1min', '5min', '15min') as $min) 331 { 332 if (isset($la_def['oid_'.$min.'_num'])) 333 { 334 $la_oids[$min] = $la_def['oid_'.$min.'_num']; 335 } 336 else if (isset($la_def['oid_'.$min])) 337 { 338 $la_oids[$min] = snmp_translate($la_def['oid_'.$min], $mib); 339 } 340 } 341 $la = snmp_get_multi_oid($device, $la_oids, array(), $mib, NULL, OBS_SNMP_ALL_NUMERIC); 342 } 343 344 print_debug_vars($la_oids); 345 print_debug_vars($la); 346 347 if (snmp_status() && is_numeric($la[$la_oids['5min']])) 348 { 349 $scale = isset($la_def['scale']) ? $la_def['scale'] : 1; 350 $scale_graph = $scale * 100; // Since want to keep compatability with old UCD-SNMP-MIB LA, graph stored as la * 100 351 352 // CLEANME after r9500, but not before CE 18.6 353 // Rename old UCD-SNMP-MIB rrds 354 if ($mib == 'UCD-SNMP-MIB') 355 { 356 rename_rrd($device, 'ucd_load.rrd', 'la.rrd'); 357 } 358 359 foreach ($la_oids as $min => $oid) 360 { 361 $device_state['la'][$min] = $la[$oid] * $scale; 362 // Now, graph specific scale if not equals 1 363 $la[$min] = $la[$oid] * $scale_graph; 364 } 365 $device_state['ucd_load'] = $device_state['la']['5min']; // Compatability witn old UCD-SNMP-MIB code 366 367 rrdtool_update_ng($device, 'la', array('1min' => $la['1min'], '5min' => $la['5min'], '15min' => $la['15min'])); 368 $graphs['la'] = TRUE; 369 370 print_cli_data('Load average', $device_state['la']['1min'].', '.$device_state['la']['5min'].', '.$device_state['la']['15min']); 371 372 break; // Stop walking other LAs 373 } 374 375 } 376} 377 378// END LA 379 380// Rewrite sysLocation if there is a mapping array or DB override 381$poll_device['sysLocation'] = snmp_fix_string($poll_device['sysLocation']); 382$poll_device['sysLocation'] = rewrite_location($poll_device['sysLocation']); 383 384if ($device['location'] != $poll_device['sysLocation']) 385{ 386 $update_array['location'] = $poll_device['sysLocation']; 387 log_event("sysLocation changed: '".$device['location']."' -> '".$poll_device['sysLocation']."'", $device, 'device', $device['device_id']); 388} 389 390$poll_device['sysContact'] = str_replace(array('\"', '"') , '', $poll_device['sysContact']); 391 392if ($poll_device['sysContact'] == 'not set') 393{ 394 $poll_device['sysContact'] = ''; 395} 396 397print_debug_vars($poll_device); 398 399// Check if snmpEngineID changed 400if (strlen($poll_device['snmpEngineID'] . $device['snmpEngineID']) && $poll_device['snmpEngineID'] != $device['snmpEngineID']) 401{ 402 $update_array['snmpEngineID'] = $poll_device['snmpEngineID']; 403 if ($device['snmpEngineID']) 404 { 405 // snmpEngineID changed frome one to other 406 log_event('snmpEngineID changed: '.$device['snmpEngineID'].' -> '.$poll_device['snmpEngineID'].' (probably the device was replaced). The device will be rediscovered.', $device, 'device', $device['device_id'], 4); 407 // Reset device discover time for full re-discovery 408 dbUpdate(array('last_discovered' => array('NULL')), 'devices', '`device_id` = ?', array($device['device_id'])); 409 } else { 410 log_event('snmpEngineID -> '.$poll_device['snmpEngineID'], $device, 'device', $device['device_id']); 411 } 412} 413 414$oids = array('sysObjectID', 'sysContact', 'sysName', 'sysDescr'); 415foreach ($oids as $oid) 416{ 417 $poll_device[$oid] = snmp_fix_string($poll_device[$oid]); 418 //print_vars($poll_device[$oid]); 419 if ($poll_device[$oid] != $device[$oid]) 420 { 421 $update_array[$oid] = ($poll_device[$oid] ? $poll_device[$oid] : array('NULL')); 422 log_event("$oid -> '".$poll_device[$oid]."'", $device, 'device', $device['device_id']); 423 424 // Update $device array for cases when model specific options required 425 if ($oid == 'sysObjectID' && $poll_device['sysObjectID']) 426 { 427 $device['sysObjectID'] = $poll_device['sysObjectID']; 428 } 429 } 430} 431 432// Restore original (not lower case) sysName 433$poll_device['sysName'] = $poll_device['sysName_original']; 434unset($poll_device['sysName_original']); 435 436print_cli_data('sysObjectID', $poll_device['sysObjectID'], 2); 437print_cli_data('snmpEngineID', $poll_device['snmpEngineID'], 2); 438print_cli_data('sysDescr', $poll_device['sysDescr'], 2); 439print_cli_data('sysName', $poll_device['sysName'], 2); 440print_cli_data('Location', $poll_device['sysLocation'], 2); 441 442// Geolocation detect 443 444$geo_detect = FALSE; 445if ($config['geocoding']['enable']) 446{ 447 $db_version = get_db_version(); // Need for detect old geo DB schema 448 449 $geo_db = dbFetchRow('SELECT * FROM `devices_locations` WHERE `device_id` = ?', array($device['device_id'])); 450 $geo_db['hostname'] = $device['hostname']; // Hostname required for detect by DNS 451 452 print_debug_vars($geo_db); 453 454 $geo_updated = $config['time']['now'] - strtotime($geo_db['location_updated']); // Seconds since previous GEO update 455 $geo_frequency = 86400; // Minimum seconds for next GEO api request (default is 1 day) 456 457 // Device coordinates still empty, redetect no more than 1 time per 1 day ($geo_frequency param) 458 if (!(is_numeric($geo_db['location_lat']) && is_numeric($geo_db['location_lon']))) 459 { 460 // Redetect geolocation if coordinates still empty, no more frequently than once a day 461 $geo_detect = $geo_detect || ($geo_updated > $geo_frequency); 462 } 463 464 // sysLocation changed (and not empty!), redetect now 465 $geo_detect = $geo_detect || ($poll_device['sysLocation'] && $device['location'] != $poll_device['sysLocation']); 466 // Geo API changed, force redetect 467 $geo_detect = $geo_detect || ($geo_db['location_geoapi'] != strtolower($config['geocoding']['api'])); 468 469 // This seems to cause endless geolocation every poll. Disabled. 470 //$geo_detect = $geo_detect || ($geo_db['location_manual'] && (!$geo_db['location_country'] || $geo_db['location_country'] == 'Unknown')); // Manual coordinates passed 471 472 // Detect location by DNS LOC record for hostname, no more than 1 time per 1 day ($geo_frequency param) 473 $dns_only = !$geo_detect && ($config['geocoding']['dns'] && ($geo_updated > $geo_frequency)); 474 $geo_detect = $geo_detect || $dns_only; 475 476 if ($geo_detect) 477 { 478 $update_geo = get_geolocation($poll_device['sysLocation'], $geo_db, $dns_only); 479 if ($update_geo) 480 { 481 print_debug_vars($update_geo, 1); 482 if (is_numeric($update_geo['location_lat']) && is_numeric($update_geo['location_lon']) && $update_geo['location_country'] != 'Unknown') 483 { 484 $geo_msg = 'Geolocation ('.strtoupper($update_geo['location_geoapi']).') -> '; 485 $geo_msg .= '['.sprintf('%f', $update_geo['location_lat']) .', ' .sprintf('%f', $update_geo['location_lon']) .'] '; 486 $geo_msg .= $update_geo['location_country'].' (Country), '.$update_geo['location_state'].' (State), '; 487 $geo_msg .= $update_geo['location_county'] .' (County), ' .$update_geo['location_city'] .' (City)'; 488 } else { 489 $geo_msg = FALSE; 490 } 491 492 if (is_numeric($geo_db['location_id'])) 493 { 494 foreach ($update_geo as $k => $value) 495 { 496 if ($geo_db[$k] == $value) { unset($update_geo[$k]); } 497 } 498 if ($update_geo) 499 { 500 dbUpdate($update_geo, 'devices_locations', '`location_id` = ?', array($geo_db['location_id'])); 501 if ($geo_msg) { log_event($geo_msg, $device, 'device', $device['device_id']); } 502 } // else not changed 503 } else { 504 $update_geo['device_id'] = $device['device_id']; 505 dbInsert($update_geo, 'devices_locations'); 506 if ($geo_msg) { log_event($geo_msg, $device, 'device', $device['device_id']); } 507 } 508 509 } 510 else if (is_numeric($geo_db['location_id'])) 511 { 512 $update_geo = array('location_updated' => format_unixtime($config['time']['now'], 'Y-m-d G:i:s')); // Increase updated time 513 dbUpdate($update_geo, 'devices_locations', '`location_id` = ?', array($geo_db['location_id'])); 514 } # end if $update_geo 515 } 516} 517 518unset($geo_detect, $geo_db, $update_geo); 519 520// EOF 521