1package dm_config; 2require Exporter; 3@ISA = qw(Exporter); 4@EXPORT = qw(initialize sync_servers time_test do_log db_connect 5 bin_path na nd db_get db_get_array db_do); 6@EXPORT_OK = qw(%c); 7 8# Devmon: An SNMP data collector & page generator for the BigBrother & 9# Hobbit network monitoring systems 10# Copyright (C) 2005-2006 Eric Schwimmer 11# Copyright (C) 2007 Francois Lacroix 12# 13# $URL: https://devmon.svn.sourceforge.net/svnroot/devmon/trunk/modules/dm_config.pm $ 14# $Revision: 120 $ 15# $Id: dm_config.pm 120 2009-01-22 18:09:01Z buchanmilne $ 16# 17# This program is free software; you can redistribute it and/or modify 18# it under the terms of the GNU General Public License as published by 19# the Free Software Foundation; either version 2 of the License, or 20# (at your option) any later version. Please see the file named 21# 'COPYING' that was included with the distrubition for more details. 22 23 24 # The global option hash. Be afraid! 25 use vars qw(%g); 26 27 # Modules 28 use strict; 29 require dm_tests; 30 require dm_templates; 31 use IO::File; 32 use FindBin; 33 34 # Load initial program values; only called once at program init 35 sub initialize { 36 37 autoflush STDOUT 1; 38 39 %g = ( 40 # General variables 41 'version' => $_[0], # set in main script now 42 'homedir' => $FindBin::Bin, 43 'configfile' => "$FindBin::Bin/devmon.cfg", 44 'dbfile' => "$FindBin::Bin/hosts.db", 45 'daemonize' => 1, 46 'initialized' => 0, 47 'mypid' => 0, 48 'verbose' => 0, 49 'debug' => 0, 50 'oneshot' => 0, 51 'print_msg' => 0, 52 'shutting_down' => 0, 53 'active' => '', 54 'pidfile' => '', 55 'logfile' => '', 56 'bbhosts' => '', 57 'bbtag' => '', 58 'bblocation' => '', 59 'nodename' => '', 60 'log' => '', 61 62 # DB variables 63 'dbhost' => '', 64 'dbname' => '', 65 'dbuser' => '', 66 'dbpass' => '', 67 'dsn' => '', 68 'dbh' => '', 69 70 # BB combo variables 71 'dispserv' => '', 72 'msgsize' => 0, 73 'msgsleep' => 0, 74 75 # Control variables 76 'my_nodenum' => 0, 77 'cycletime' => 0, 78 'deadtime' => 0, 79 'parent' => 1, 80 'numforks' => 0, 81 'forks' => {}, 82 'maxpolltime' => 30, 83 'maxreps' => 10, 84 85 # Statistical vars 86 'numdevs' => 0, 87 'numtests' => 0, 88 'avgtestsnode' => 0, 89 'snmppolltime' => 0, 90 'testtime' => 0, 91 'msgxfrtime' => 0, 92 'numclears' => {}, 93 'avgpolltime' => [], 94 95 # SNMP variables 96 'snmptimeout' => 0, 97 'snmptries' => 0, 98 'snmpcids' => '', 99 100 # Now our global data subhashes 101 'templates' => {}, 102 'dev_data' => {}, 103 'dev_hist' => {}, 104 'tmp_hist' => {}, 105 'clear_data' => {}, 106 'snmp_data' => {}, 107 'fails' => {}, 108 'max_rep_hist' => {}, 109 'node_status' => {}, 110 'hobbit_color' => {}, 111 'test_results' => [], 112 113 # User-definable variable controls 114 'globals' => {}, 115 'locals' => {} 116 ); 117 118 # Our local options 119 %{$g{'locals'}} = ( 120 'multinode' => { 'default' => 'no', 121 'regex' => 'yes|no', 122 'set' => 0, 123 'case' => 0 }, 124 'bbhosts' => { 'default' => (defined $ENV{'BBHOSTS'} and $ENV{'BBHOSTS'} ne '') ? $ENV{'BBHOSTS'} : '/home/hobbit/server/etc/bb-hosts', 125 'regex' => '.+', 126 'set' => 0, 127 'case' => 1 }, 128 'bblocation' => { 'default' => (defined $ENV{'BBLOCATION'} and $ENV{'BBLOCATION'} ne '') ? $ENV{'BBLOCATION'} : '', 129 'regex' => '\w+', 130 'set' => 0, 131 'case' => 1 }, 132 'bbtag' => { 'default' => 'DEVMON', 133 'regex' => '\w+', 134 'set' => 0, 135 'case' => 1 }, 136 'snmpcids' => { 'default' => 'public,private', 137 'regex' => '\S+', 138 'set' => 0, 139 'case' => 1 }, 140 'nodename' => { 'default' => 'HOSTNAME', 141 'regex' => '[\w\.-]+', 142 'set' => 0, 143 'case' => 1 }, 144 'pidfile' => { 'default' => '/var/run/devmon/devmon.pid', 145 'regex' => '.+', 146 'set' => 0, 147 'case' => 1 }, 148 'logfile' => { 'default' => '/var/log/devmon.log', 149 'regex' => '.*', 150 'set' => 0, 151 'case' => 1 }, 152 'dbhost' => { 'default' => 'localhost', 153 'regex' => '\S+', 154 'set' => 0, 155 'case' => 0 }, 156 'dbname' => { 'default' => 'devmon', 157 'regex' => '\w+', 158 'set' => 0, 159 'case' => 1 }, 160 'dbuser' => { 'default' => 'devmon', 161 'regex' => '\w+', 162 'set' => 0, 163 'case' => 1 }, 164 'dbpass' => { 'default' => 'devmon', 165 'regex' => '\S+', 166 'set' => 0, 167 'case' => 1 } 168 ); 169 170 # Our global options 171 %{$g{'globals'}} = ( 172 'bbtype' => { 'default' => 'hobbit', 173 'regex' => 'bb|hobbit', 174 'set' => 0, 175 'case' => 0 }, 176 'dispserv' => { 'default' => (defined $ENV{'BBDISP'} and $ENV{BBDISP} ne '') ? $ENV{'BBDISP'} : 'localhost', 177 'regex' => '\S+', 178 'set' => 0, 179 'case' => 0 }, 180 'dispport' => { 'default' => (defined $ENV{'BBPORT'} and $ENV{'BBPORT'} ne '') ? $ENV{'BBPORT'} : 1984, 181 'regex' => '\d+', 182 'set' => 0, 183 'case' => 0 }, 184 'bbdateformat' => { 'default' => (defined $ENV{'BBDATEFORMAT'} and $ENV{'BBDATEFORMAT'} ne '') ? $ENV{'BBDATEFORMAT'} : '', 185 'regex' => '.+', 186 'set' => 0, 187 'case' => 1 }, 188 'msgsize' => { 'default' => 8096, 189 'regex' => '\d+', 190 'set' => 0, 191 'case' => 0 }, 192 'msgsleep' => { 'default' => 10, 193 'regex' => '\d+', 194 'set' => 0, 195 'case' => 0 }, 196 'cycletime' => { 'default' => 60, 197 'regex' => '\d+', 198 'set' => 0, 199 'case' => 0 }, 200 'cleartime' => { 'default' => 180, 201 'regex' => '\d+', 202 'set' => 0, 203 'case' => 0 }, 204 'deadtime' => { 'default' => 60, 205 'regex' => '\d+', 206 'set' => 0, 207 'case' => 0 }, 208 'numforks' => { 'default' => 10, 209 'regex' => '\d+', 210 'set' => 0, 211 'case' => 0 }, 212 'maxpolltime' => { 'default' => 30, 213 'regex' => '\d+', 214 'set' => 0, 215 'case' => 0 }, 216 'snmptimeout' => { 'default' => 2, 217 'regex' => '\d+', 218 'set' => 0, 219 'case' => 0 }, 220 'snmptries' => { 'default' => 5, 221 'regex' => '\d+', 222 'set' => 0, 223 'case' => 0 }, 224 ); 225 226 # Set up our signal handlers 227 $SIG{INT} = $SIG{QUIT} = $SIG{TERM} = \&quit; 228 $SIG{HUP} = \&reopen_log; 229 230 # Parse command line options 231 my($syncconfig,$synctemps,$resetowner,$readhosts); 232 $syncconfig = $synctemps = $resetowner = $readhosts = 0; 233 while ($_ = shift @ARGV) { 234 if(/^-v+$/) { $g{'verbose'} = tr/v/v/ } 235 elsif(/^-c$/) { $g{'configfile'} = shift @ARGV or usage() } 236 elsif(/^-d/) { $g{'dbfile'} = shift @ARGV or usage() } 237 elsif(/^-f$/) { $g{'daemonize'} = 0 } 238 elsif(/^-p$/) { $g{'print_msg'} = 1 } 239 elsif(/^-1$/) { $g{'print_msg'} = 1; 240 $g{'verbose'} = 2; 241 $g{'debug'} = 1; 242 $g{'oneshot'} = 1 } 243 elsif(/^--debug$/) { $g{'debug'} = 1 } 244 elsif(/^--syncconfig$/) { $syncconfig = 1 } 245 elsif(/^--synctemplates$/) { $synctemps = 1 } 246 elsif(/^--resetowners$/) { $resetowner = 1 } 247 elsif(/^--readbbhosts$/) { $readhosts = 1 } 248 else { usage () } 249 } 250 251 # Now read in our local config info from our file 252 read_local_config(); 253 254 # Prevent multiple mutually exclusives 255 if($syncconfig + $synctemps + $resetowner + $readhosts > 1) { 256 print "Can't have more than one mutually exclusive option\n\n"; 257 usage(); 258 } 259 260 # Check mutual exclusions (run-and-die options) 261 sync_global_config() if $syncconfig; 262 dm_templates::sync_templates() if $synctemps; 263 reset_ownerships() if $resetowner; 264 read_bb_hosts() if $readhosts; 265 266 267 268 # Dont daemonize if we are printing messages or debug 269 $g{'daemonize'} = 0 if $g{'print_msg'} or $g{'debug'}; 270 271 # Daemonize if need be 272 daemonize(); 273 274 # Open the log file 275 open_log(); 276 277 # Set our pid 278 $g{'mypid'} = $$; 279 280 # Check to see if a pid file exists 281 if(-e $g{'pidfile'}) { 282 # One exists, let see if its stale 283 my $pid_handle = new IO::File $g{'pidfile'}, 'r' 284 or log_fatal("Can't read from pid file '$g{'pidfile'}' ($!).", 0); 285 286 # Read in the old PID 287 my ($old_pid) = <$pid_handle>; 288 chomp $old_pid; 289 $pid_handle->close; 290 291 # If it exists, die silently 292 log_fatal("Devmon already running, quitting.", 1) if kill 0, $old_pid; 293 } 294 295 # Now write our pid to the pidfile 296 if(!$g{'debug'}) { 297 my $pid_handle = new IO::File $g{'pidfile'}, 'w' 298 or log_fatal("Cant write to pidfile $g{'pidfile'} ($!)",0); 299 $pid_handle->print($g{'mypid'}); 300 $pid_handle->close; 301 } 302 303 # Autodetect our nodename on user request 304 if($g{'nodename'} eq 'HOSTNAME') { 305 my $hostname_bin = bin_path('hostname'); 306 307 die "Unable to find 'hostname' command!\n" if !defined $hostname_bin; 308 my $nodename = `$hostname_bin`; 309 chomp $nodename; 310 die "Error executing $hostname_bin" 311 if $nodename =~ /not found|permission denied/i; 312 313 # Remove domain info, if any 314 # Hobbit best practice is to use fqdn, if the user doesnt want it 315 # we assume they have set NODENAME correctly in devmon.cfg 316 # $nodename =~ s/\..*//; 317 chomp $nodename; 318 319 $g{'nodename'} = $nodename; 320 321 do_log("Nodename autodetected as $nodename", 2); 322 } 323 324 # Make sure we have a nodename 325 die "Unable to determine nodename!\n" if !defined $g{'nodename'} 326 and $g{'nodename'} =~ /^\S+$/; 327 328 # Set up DB handle 329 db_connect(1); 330 331 # Connect to the cluster 332 cluster_connect(); 333 334 # Read our global variables 335 read_global_config(); 336 337 # Throw out a little info to the log 338 do_log("---Initilizing devmon...",0); 339 do_log("Verbosity level: $g{'verbose'}",1); 340 do_log("Logging to $g{'logfile'}",1); 341 do_log("Node $g{'my_nodenum'} reporting to $g{'dispserv'}",0); 342 do_log("Running under process id: $g{'mypid'}",0); 343 344 # We are now initialized 345 $g{'initialized'} = 1; 346 } 347 348 349 350 # Determine the amount of time we spent doing tests 351 sub time_test { 352 353 do_log("DEBUG CFG: running time_test()",0) if $g{'debug'}; 354 355 my $poll_time = $g{'snmppolltime'} + $g{'testtime'} + $g{'msgxfrtime'}; 356 357 # Add our current poll time to our history array 358 push @{$g{'avgpolltime'}}, $poll_time; 359 while (@{$g{'avgpolltime'}} > 5) { 360 shift @{$g{'avgpolltime'}} # Only preserve 5 entries 361 } 362 363 # Squak if we went over our poll time 364 my $exceeded = $poll_time - $g{'cycletime'}; 365 if($exceeded > 1) { 366 do_log("Exceeded cycle time ($poll_time seconds).", 0); 367 $g{'sleep_time'} = 0; 368 quit(0) if $g{'oneshot'}; 369 } 370 371 # Otherwise calculate our sleep time 372 else { 373 quit(0) if $g{'oneshot'}; 374 $g{'sleep_time'} = -$exceeded; 375 $g{'sleep_time'} = 0 if $g{'sleep_time'} < 0; # just in case! 376 do_log("Sleeping for $g{'sleep_time'} seconds.", 1); 377 sleep $g{'sleep_time'} if $g{'sleep_time'}; 378 } 379 } 380 381 382 383 # Subroutine to reload test data when needed and handle fail-over 384 sub sync_servers { 385 386 my %device_hash; 387 my %available_devices; 388 my %test_count; 389 my %custom_threshs; 390 my %custom_excepts; 391 my ($total_tests, $my_num_tests, $need_init); 392 393 # If we are multinode='no', just load our tests and return 394 if($g{'multinode'} ne 'yes') { 395 %{$g{'dev_data'}} = read_hosts(); 396 return; 397 398 } 399 400 # First things first, update heartbeat info 401 db_do("update nodes set heartbeat='" . time . 402 "' where node_num=" . $g{'my_nodenum'}); 403 404 # Reload our global config 405 read_global_config(); 406 407 # Read in all node configuration data 408 update_nodes(); 409 410 # If someone has set our flag to inactive, quietly die 411 if($g{'node_status'}{'nodes'}{$g{'my_nodenum'}}{'active'} ne 'y') { 412 do_log("Active flag has been set to a non-true value. Exiting.", 0); 413 exit 0; 414 } 415 416 # See if we need to read our templates 417 if($g{'node_status'}{'nodes'}{$g{'my_nodenum'}}{'read_temps'} eq 'y') { 418 dm_templates::read_templates(); 419 $g{'node_status'}{'nodes'}{$g{'my_nodenum'}}{'read_temps'} = 'n'; 420 } 421 422 # We need an init by default, but if anybody has any tests, set to 0 423 $need_init = 1; 424 425 %{$g{'dev_data'}} = (); 426 427 # Assume we have 0 tests to begin with 428 $my_num_tests = 0; 429 $total_tests = 0; 430 431 # Read in all custom thresholds 432 my @threshs 433 = db_get_array('host,test,oid,color,val from custom_threshs'); 434 for my $this_thresh (@threshs) { 435 my ($host, $test, $oid, $color, $val) = @$this_thresh; 436 437 $custom_threshs{$host}{$test}{$oid}{$color} = $val; 438 } 439 440 # Read in all custom exceptions 441 my @excepts 442 = db_get_array('host,test,oid,type,data from custom_excepts'); 443 for my $this_except (@excepts) { 444 my ($host, $test, $oid, $type, $data) = @$this_except; 445 $custom_excepts{$host}{$test}{$oid}{$type} = $data; 446 } 447 448 # Read in all tests for all nodes 449 my @tests 450 = db_get_array('name,ip,vendor,model,tests,cid,owner from devices'); 451 for my $this_test (@tests) { 452 my ($device, $ip, $vendor, $model, $tests, $cid, $owner) = @$this_test; 453 454 # Make sure we disable our init if someone already has a test 455 if($owner != 0) {$need_init = 0} 456 457 $device_hash{$device}{'ip'} = $ip; 458 $device_hash{$device}{'vendor'} = $vendor; 459 $device_hash{$device}{'model'} = $model; 460 $device_hash{$device}{'tests'} = $tests; 461 $device_hash{$device}{'cid'} = $cid; 462 $device_hash{$device}{'owner'} = $owner; 463 464 # Do some numerical accounting that we use to load-balance later 465 466 # Determine the number of tests that this host has 467 my $dev_tests; 468 if($tests eq 'all') { 469 $dev_tests = scalar keys %{$g{'templates'}{$vendor}{$model}{'tests'}}; 470 } 471 else { 472 $dev_tests = ($tests =~ tr/,/,/) + 1; 473 } 474 475 $total_tests += $dev_tests; 476 $test_count{$device} = $dev_tests; 477 478 # If this test is ours, claim it! 479 if ($owner == $g{'my_nodenum'}) { 480 $my_num_tests += $dev_tests; 481 $g{'dev_data'}{$device} = $device_hash{$device}; 482 %{$g{'dev_data'}{$device}{'thresh'}} = %{$custom_threshs{$device}} 483 if defined $custom_threshs{$device}; 484 %{$g{'dev_data'}{$device}{'except'}} = %{$custom_excepts{$device}} 485 if defined $custom_excepts{$device}; 486 } 487 488 # If this test doesnt have an owner, lets add it to the available pool 489 if($owner == 0 or not defined $g{'node_status'}{'active'}{$owner}) { 490 push @{$available_devices{$dev_tests}}, $device; 491 } 492 } 493 494 # Determine our number of active nodes 495 my @active_nodes = sort na keys %{$g{'node_status'}{'active'}}; 496 my $num_active_nodes = @active_nodes + 0; 497 498 # Determine the avg number of tests/node 499 my $avg_tests_node = int $total_tests / $num_active_nodes; 500 501 # Now lets see if we need tests 502 if($my_num_tests < $avg_tests_node) { 503 504 # First, let evertbody know that we need tests 505 my $num_tests_needed = $avg_tests_node - $my_num_tests; 506 db_do("update nodes set need_tests=$num_tests_needed " . 507 "where node_num=$g{'my_nodenum'}"); 508 509 # Lets see if we need to init, along with the other nodes 510 if($need_init) { 511 do_log("Initializing test database",0); 512 # Now we need all other nodes waiting for init before we can proceed 513 do_log("Waiting for all nodes to synchronize",0); 514 INIT_WAIT: while(1) { 515 516 # Make sure our heart beats while we wait 517 db_do("update nodes set heartbeat='" . time . 518 "' where node_num='$g{'my_nodenum'}'"); 519 520 for my $node (keys %{$g{'node_status'}{'active'}}) { 521 next if $node == $g{'my_nodenum'}; 522 if ($g{'node_status'}{'nodes'}{$node}{'need_tests'} 523 != $avg_tests_node) { 524 my $name = $g{'node_status'}{'nodes'}{$node}{'name'}; 525 # This node isnt ready for init; sleep then try again 526 do_log("Waiting for node $node($name)",0); 527 sleep 2; 528 update_nodes(); 529 next INIT_WAIT; 530 } 531 } 532 533 # Looks like all nodes are ready, exit the loop 534 sleep 2; 535 last; 536 } 537 do_log("Done waiting",0); 538 539 # Now assign all tests using a round-robin technique; this should 540 # synchronize the tests between all servers 541 my @available; 542 for my $count (sort nd keys %available_devices) { 543 push @available, @{$available_devices{$count}}; 544 } 545 546 @active_nodes = sort na keys %{$g{'node_status'}{'active'}}; 547 $num_active_nodes = @active_nodes + 0; 548 $avg_tests_node = int $total_tests / $num_active_nodes; 549 550 my $this_node = 0; 551 for my $device (@available) { 552 # Skip any test unless the count falls on our node num 553 if($active_nodes[$this_node++] == $g{'my_nodenum'}) { 554 # Make it ours, baby! 555 my $result = db_do("update devices set " . 556 "owner=$g{'my_nodenum'} where name='$device' and owner=0"); 557 # Make sure out DB update went through 558 next if !$result; 559 # Now stick the pertinent data in our variables 560 $my_num_tests += $test_count{$device}; 561 $g{'dev_data'}{$device} = $device_hash{$device}; 562 %{$g{'dev_data'}{$device}{'thresh'}} = %{$custom_threshs{$device}} 563 if defined $custom_threshs{$device}; 564 %{$g{'dev_data'}{$device}{'except'}} = %{$custom_excepts{$device}} 565 if defined $custom_excepts{$device}; 566 } 567 568 # Make sure we arent out of bounds 569 $this_node = 0 if $this_node > $#active_nodes; 570 } 571 572 do_log("Init complete: $my_num_tests tests loaded, " . 573 "avg $avg_tests_node tests per node", 0); 574 } 575 576 # Okay, we're not at init, so lets see if we can find any available tests 577 else { 578 579 for my $count (sort nd keys %available_devices) { 580 581 # Go through all the devices for this test count 582 for my $device (@{$available_devices{$count}}) { 583 584 # Make sure we havent hit our limit 585 last if $my_num_tests > $avg_tests_node; 586 587 # Lets try and take this test 588 my $result = db_do("update devices set " . 589 "owner=$g{'my_nodenum'} where name='$device'"); 590 next if !$result; 591 592 # We got it! Lets add it to our test_data hash 593 $my_num_tests += $count; 594 my $old_owner = $device_hash{$device}{'owner'}; 595 596 # Add data to our hashes 597 $g{'dev_data'}{$device} = $device_hash{$device}; 598 %{$g{'dev_data'}{$device}{'thresh'}} = %{$custom_threshs{$device}} 599 if defined $custom_threshs{$device}; 600 %{$g{'dev_data'}{$device}{'except'}} = %{$custom_excepts{$device}} 601 if defined $custom_excepts{$device}; 602 603 # Log where this device came from 604 if($old_owner == 0) { 605 do_log("Got $device ($my_num_tests/$avg_tests_node tests)"); 606 } 607 else { 608 my $old_name = $g{'node_status'}{'nodes'}{$old_owner}{'name'}; 609 $old_name = "unknown" if !defined $old_name; 610 do_log("Recovered $device from node $old_owner($old_name) " . 611 "($my_num_tests/$avg_tests_node tests)",0); 612 } 613 614 # Now lets try and get the history for it, if it exists 615 my @hist_arr = 616 db_get_array('ifc,test,time,val from test_data ' . 617 "where host='$device'"); 618 for my $hist (@hist_arr) { 619 my ($ifc,$test,$time,$val) = @$hist; 620 $g{'dev_hist'}{$device}{$ifc}{$test}{'val'} = $val; 621 $g{'dev_hist'}{$device}{$ifc}{$test}{'time'} = $time; 622 } 623 624 # Now delete it from the history table 625 db_do("delete from test_data where host='$device'"); 626 } 627 } 628 } 629 630 # Now lets update the DB with how many tests we still need 631 $num_tests_needed = $avg_tests_node - $my_num_tests; 632 $num_tests_needed = 0 if $num_tests_needed < 0; 633 db_do("update nodes set need_tests=$num_tests_needed " . 634 "where node_num=$g{'my_nodenum'}"); 635 636 } 637 638 # If we dont need any tests, lets see if we can donate any tests 639 elsif ($my_num_tests > $avg_tests_node) { 640 641 my $tests_they_need; 642 my $biggest_test_needed = 0; 643 644 # Read in the number of needy nodes 645 for my $this_node (@active_nodes) { 646 next if $this_node == $g{'my_nodenum'}; 647 my $this_node_needs = 648 $g{'node_status'}{'nodes'}{$this_node}{'need_tests'}; 649 $tests_they_need += $this_node_needs; 650 $biggest_test_needed = $this_node_needs 651 if $this_node_needs > $biggest_test_needed; 652 } 653 654 # Now go through the devices and assign any I can 655 for my $device (keys %{$g{'dev_data'}}) { 656 # Make sure this test isnt too big 657 next if $test_count{$device} > $biggest_test_needed 658 659 # Now make sure that it wont put us under the avg_nodes 660 or $my_num_tests - $test_count{$device} <= $avg_tests_node; 661 662 # Okay, lets assign it to the open pool, then 663 my $result = db_do("update devices set owner=0 where " . 664 "name='$device' and owner=$g{'my_nodenum'}"); 665 666 # We really shouldnt fail this, but just in case 667 next if !$result; 668 $my_num_tests -= $test_count{$device}; 669 do_log("Dropped $device ($my_num_tests/$avg_tests_node tests)", 0); 670 671 # Now stick the history for the device in the DB for the recipient 672# for my $ifc (keys %{$g{'dev_hist'}{$device}}) { 673# for my $test (keys %{$g{'dev_hist'}{$device}{$ifc}}) { 674# my $val = $g{'dev_hist'}{$device}{$ifc}{$test}{'val'}; 675# my $time = 0; 676# if(defined $g{'dev_hist'}{$device}{$ifc}{$test}{'time'}) { 677# $time = $g{'dev_hist'}{$device}{$ifc}{$test}{'time'}; 678# } 679# 680# db_do("insert into test_data (host,ifc,test,time,val) " . 681# "values ('$device','$ifc','$test',$time,'$val')"); 682# } 683# } 684 685 # Now delete the test from our hash 686 delete $g{'dev_data'}{$device}; 687 } 688 } 689 690 # Record some statistics 691 $g{'numtests'} = $my_num_tests; 692 $g{'avgtestsnode'} = $avg_tests_node; 693 $g{'numdevs'} = scalar keys %{$g{'dev_data'}}; 694 695 } 696 697 698 699 # Sub to update node status & configuration 700 sub update_nodes { 701 702 # Make a copy of our node status 703 my %old_status = %{$g{'node_status'}}; 704 705 %{$g{'node_status'}} = (); 706 my @nodes = db_get_array('name,node_num,active,heartbeat,need_tests,' . 707 'read_temps from nodes'); 708 709 NODE: for my $node (@nodes) { 710 my ($name, $node_num, $active, $heartbeat, $need_tests, 711 $read_temps) = @$node; 712 $g{'node_status'}{'nodes'}{$node_num} = { 713 'name' => $name, 714 'active' => $active, 715 'heartbeat' => $heartbeat, 716 'need_tests' => $need_tests, 717 'read_temps' => $read_temps 718 }; 719 720 # Check to see if its inactive 721 if($active ne 'y') { 722 $g{'node_status'}{'inactive'}{$node_num} = 1; 723 next NODE; 724 } 725 726 # Check to see if this host has died (i.e. exceeded deadtime) 727 elsif($heartbeat + $g{'deadtime'} < time) { 728 do_log("Node $node_num($name) has died!") 729 if !defined $old_status{'dead'}{$node_num}; 730 $g{'node_status'}{'dead'}{$node_num} = time; 731 } 732 733 # Now check and see if it was previously dead and has returned 734 elsif(defined $old_status{'dead'}{$node_num}) { 735 my $up_duration = time - $old_status{'dead'}{$node_num}; 736 if ($up_duration > ($g{'deadtime'} * 2)) { 737 $g{'node_status'}{'active'}{$node_num} = 1; 738 do_log("Node $node_num($name) has returned! " . 739 "Up $up_duration secs",0); 740 } 741 else { 742 $g{'node_status'}{'dead'}{$node_num} 743 = $old_status{'dead'}{$node_num}; 744 } 745 } 746 747 # If it passed, add it to the active sub-hash 748 else { 749 $g{'node_status'}{'active'}{$node_num} = 1; 750 } 751 } 752 } 753 754 755 756 # Connect our node to the cluster 757 # Basically this means just updated the nodes table in the database 758 # So that our node is listed as active and we have a current heartbeat 759 sub cluster_connect { 760 761 # Dont bother if we arent multinode 762 return if $g{'multinode'} ne 'yes'; 763 764 my $now = time; 765 my $nodenum; 766 my $nodename = $g{'nodename'}; 767 my %nodes; 768 769 # First pull down all our node info to make sure we exist in the table 770 my @nodeinfo = db_get_array("name,node_num from nodes"); 771 772 for my $row (@nodeinfo) { 773 my ($name, $num) = @$row; 774 $nodes{$num} = $name; 775 $nodenum = $num if $name eq $nodename; 776 } 777 778 # If we arent in the table, lets add ourself 779 if(!defined $nodenum) { 780 781 # Find the next available num 782 my $ptr; 783 while (!defined $nodenum) { 784 $nodenum = $ptr if !defined $nodes{++$ptr}; 785 } 786 787 # Do the db add 788 db_do("insert into nodes values ('$nodename',$nodenum,'y',$now,0,'n')"); 789 } 790 791 # If we are in the table, update our activity and heartbeat columns 792 else { 793 db_do("update nodes set active='y', heartbeat=$now " . 794 "where node_num=$nodenum" ); 795 } 796 797 # Set our global nodenum 798 $g{'my_nodenum'} = $nodenum; 799 800 } 801 802 803 804 # Sub to load/reload global configuration data 805 sub read_global_config { 806 if($g{'multinode'} eq 'yes') { read_global_config_db() } 807 else { read_global_config_file() } 808 } 809 810 811 812 # Read in the local config parameters from the config file 813 sub read_local_config { 814 815 # Open config file (assuming we can find it) 816 my $file = $g{'configfile'}; 817 &usage if !defined $file; 818 819 if ($file !~ /^\/.+/ and !-e $file) { 820 my $local_file = $FindBin::Bin . "/$file"; 821 $file = $local_file if -e $local_file; 822 } 823 824 log_fatal("Can't find config file $file ($!)",0) if !-e $file; 825 open FILE, $file or log_fatal("Can't read config file $file ($!)",0); 826 827 # Parse file text 828 for my $line (<FILE>) { 829 830 # Skip empty lines and comments 831 next if $line =~ /^\s*(#.*)?$/; 832 833 chomp $line; 834 my ($option, $value) = split /\s*=\s*/, $line, 2; 835 836 # Make sure we have option and value 837 log_fatal("Syntax error in config file at line $.",0) 838 if !defined $option or !defined $value; 839 840 # Options are case insensitive 841 $option = lc $option; 842 843 # Skip global options 844 next if defined $g{'globals'}{$option}; 845 846 # Croak if this option is unknown 847 log_fatal("Unknown option '$option' in config file, line $.",0) 848 if !defined $g{'locals'}{$option}; 849 850 # If this option isnt case sensitive, lowercase it 851 $value = lc $value if !$g{'locals'}{$option}{'case'}; 852 853 # Compare to regex, make sure value is valid 854 log_fatal("Invalid value '$value' for '$option' in config file, " . 855 "line $.",0) 856 if $value !~ /^$g{'locals'}{$option}{'regex'}$/; 857 858 # Assign the value to our option 859 $g{$option} = $value; 860 $g{'locals'}{$option}{'set'} = 1; 861 } 862 close FILE; 863 864 # Log any options not set 865 for my $opt (sort keys %{$g{'locals'}}) { 866 next if $g{'locals'}{$opt}{'set'}; 867 do_log("Option '$opt' defaulting to: $g{'locals'}{$opt}{'default'}",2); 868 $g{$opt} = $g{'locals'}{$opt}{'default'}; 869 $g{'locals'}{$opt}{'set'} = 1; 870 } 871 872 # Set DSN 873 $g{'dsn'} = 'DBI:mysql:' . $g{'dbname'} . ':' . $g{'dbhost'}; 874 875 } 876 877 878 879 # Read global config from file (as oppsed to db) 880 sub read_global_config_file { 881 882 # Open config file (assuming we can find it) 883 my $file = $g{'configfile'}; 884 log_fatal("Can't find config file $file ($!)",0) if !-e $file; 885 886 open FILE, $file or log_fatal("Can't read config file $file ($!)", 0); 887 888 # Parse file text 889 for my $line (<FILE>) { 890 891 # Skip empty lines and comments 892 next if $line =~ /^\s*(#.*)?$/; 893 894 chomp $line; 895 my ($option, $value) = split /\s*=\s*/, $line, 2; 896 897 # Make sure we have option and value 898 log_fatal("Syntax error in config file at line $.",0) 899 if !defined $option or !defined $value; 900 901 # Options are case insensitive 902 $option = lc $option; 903 904 # Skip local options 905 next if defined $g{'locals'}{$option}; 906 907 # Croak if this option is unknown 908 log_fatal("Unknown option '$option' in config file, line $.",0) 909 if !defined $g{'globals'}{$option}; 910 911 # If this option isnt case sensitive, lowercase it 912 $value = lc $value if !$g{'globals'}{$option}{'case'}; 913 914 # Compare to regex, make sure value is valid 915 log_fatal("Invalid value '$value' for '$option' in config file, " . 916 "line $.", 0) 917 if $value !~ /^$g{'globals'}{$option}{'regex'}$/; 918 919 # Assign the value to our option 920 $g{$option} = $value; 921 $g{'globals'}{$option}{'set'} = 1; 922 } 923 924 # Log any options not set 925 for my $opt (sort keys %{$g{'globals'}}) { 926 next if $g{'globals'}{$opt}{'set'}; 927 do_log("Option '$opt' defaulting to: $g{'globals'}{$opt}{'default'}.",2); 928 $g{$opt} = $g{'globals'}{$opt}{'default'}; 929 $g{'globals'}{$opt}{'set'} = 1; 930 } 931 932 close FILE; 933 } 934 935 936 937 # Read global configuration from the DB 938 sub read_global_config_db { 939 my %old_globals; 940 941 # Store our old variables, then unset them 942 for my $opt (keys %{$g{'globals'}}) { 943 $old_globals{$opt} = $g{$opt}; 944 $g{'globals'}{$opt}{'set'} = 0; 945 } 946 947 my @variable_arr = db_get_array('name,val from global_config'); 948 for my $variable (@variable_arr) { 949 my ($opt,$val) = @$variable; 950 do_log("Unknown option '$opt' read from global DB") and next 951 if !defined $g{'globals'}{$opt}; 952 do_log("Invalid value '$val' for '$opt' in global DB") and next 953 if $val !~ /$g{'globals'}{$opt}{'regex'}/; 954 955 $g{'globals'}{$opt}{'set'} = 1; 956 $g{$opt} = $val; 957 } 958 959 # If we have any variables whose values have changed, write to DB 960 my $rewrite_config = 0; 961 if($g{'initialized'}) { 962 for my $opt (keys %{$g{'globals'}}) { 963 $rewrite_config = 1 if $g{$opt} ne $old_globals{$opt}; 964 } 965 } 966 rewrite_config() if $rewrite_config; 967 968 # Make sure nothing was missed 969 for my $opt (keys %{$g{'globals'}}) { 970 next if $g{'globals'}{$opt}{'set'}; 971 do_log("Option '$opt' defaulting to: $g{'globals'}{$opt}{'default'}.",2); 972 $g{$opt} = $g{'globals'}{$opt}{'default'}; 973 $g{'globals'}{$opt}{'set'} = 1; 974 } 975 976 } 977 978 979 980 # Rewrite the config file if we have seen a change in the global DB 981 sub rewrite_config { 982 my @text_out; 983 984 # Open config file (assuming we can find it) 985 my $file = $g{'configfile'}; 986 log_fatal("Can't find config file $file ($!)",0) if !-e $file; 987 988 open FILE, $file or log_fatal("Can't read config file $file ($!)", 0); 989 my @file_text = <FILE>; 990 close FILE; 991 992 for my $line (@file_text) { 993 next if $line !~ /^\s*(\S+)=(.+)$/; 994 my ($opt, $val) = split '=', $line; 995 my $new_val = $g{$opt}; 996 $line =~ s/=$val/=$new_val/; 997 push @text_out, $line; 998 } 999 1000 open FILE, ">$file" or 1001 log_fatal("Can't write to config file $file ($!)",0) if !-e $file; 1002 for my $line (@text_out) {print FILE $line} 1003 close FILE; 1004 } 1005 1006 1007 1008 # Open log file 1009 sub open_log { 1010 1011 # Dont open the log if we are in debug mode 1012 return if $g{'logfile'} =~ /^\s*$/ or $g{'debug'}; 1013 1014 $g{'log'} = new IO::File $g{'logfile'}, 'a' 1015 or log_fatal("Unable to open logfile! ($!)",0); 1016 $g{'log'}->autoflush(1); 1017 } 1018 1019 # Allow Rotation of log files 1020 sub reopen_log { 1021 my ($signal) = @_; 1022 do_log("Received signal $signal, closing and re-opening log file",3); 1023 if (defined $g{'log'}) { 1024 undef $g{'log'}; 1025 &open_log; 1026 } 1027 do_log("Re-opened log file $g{'logfile'}",3); 1028 return 1; 1029 } 1030 1031 1032 # Sub to log data to a logfile and print to screen if verbose 1033 sub do_log { 1034 my ($msg, $verbosity) = @_; 1035 1036 $verbosity = 0 if !defined $verbosity; 1037 my $ts = ts(); 1038 if (defined $g{'log'} and $g{'log'} ne '') { 1039 $g{'log'}->print("$ts $msg\n") if $g{'verbose'} >= $verbosity;; 1040 } 1041 else { 1042 print "$ts $msg\n" if $g{'verbose'} >= $verbosity;; 1043 return 1; 1044 } 1045 1046 print "$ts $msg\n" if $g{'verbose'} > $verbosity; 1047 1048 return 1; 1049 } 1050 1051 1052 1053 # Log and die 1054 sub log_fatal { 1055 my ($msg, $verbosity,$exitcode) = @_; 1056 1057 do_log($msg, $verbosity); 1058 quit(1); 1059 } 1060 1061 1062 1063 # Sub to make a nice timestamp 1064 sub ts { 1065 my ($sec,$min,$hour,$day,$mon,$year) = localtime; 1066 sprintf '[%-2.2d-%-2.2d-%-2.2d@%-2.2d:%-2.2d:%-2.2d]', $year-100, $mon+1, 1067 $day, $hour, $min, $sec, 1068 } 1069 1070 1071 1072 # Connect/recover DB connection 1073 sub db_connect { 1074 my ($silent) = @_; 1075 1076 # Dont need this if we are not in multinode mode 1077 return if $g{'multinode'} ne 'yes'; 1078 1079 # Load the DBI module if we havent initilized yet 1080 if(!$g{'initilized'}) { 1081 require DBI if !$g{'initilized'}; 1082 DBI->import(); 1083 } 1084 1085 do_log("Connecting to DB",2) if !defined $silent; 1086 $g{'dbh'}->disconnect() if defined $g{'dbh'} and $g{'dbh'} ne ''; 1087 1088 # 5 connect attempts 1089 my $try; 1090 for (1 .. 5) { 1091 $g{'dbh'} = DBI->connect( 1092 $g{'dsn'}, 1093 $g{'dbuser'}, 1094 $g{'dbpass'}, 1095 {AutoCommit => 1, RaiseError => 0, PrintError => 1} 1096 ) and return; 1097 1098 # Sleep 12 seconds 1099 sleep 12; 1100 1101 do_log("Failed to connect to DB, attempt ".++$try." of 5",0); 1102 } 1103 print "Verbose: ", $g{'verbose'}, "\n"; 1104 do_log("ERROR: Unable to connect to DB ($!)",0); 1105 } 1106 1107 1108 1109 # Sub to query DB, return results, die if error 1110 sub db_get { 1111 my ($query) = @_; 1112 do_log("DEBUG DB: select $query") if $g{'debug'}; 1113 my @results; 1114 my $a = $g{'dbh'}->selectall_arrayref("select $query") or 1115 do_log("DB query '$query' failed; reconnecting",0) 1116 and db_connect() 1117 and return db_get($query); 1118 1119 for my $b (@$a) { 1120 for my $c (@$b) { 1121 push @results, $c; 1122 } 1123 } 1124 1125 return @results; 1126 } 1127 1128 1129 1130 # Sub to query DB, return resulting array, die if error 1131 sub db_get_array { 1132 my ($query) = @_; 1133 do_log("DEBUG DB: select $query") if $g{'debug'}; 1134 my $results = $g{'dbh'}->selectall_arrayref("select $query") or 1135 do_log("DB query '$query' failed; reconnecting",0) 1136 and db_connect() 1137 and return db_get_array($query); 1138 1139 return @$results; 1140 } 1141 1142 1143 1144 # Sub to write to db, die if error 1145 sub db_do { 1146 my ($cmd) = @_; 1147 1148 # Make special characters mysql safe 1149 $cmd =~ s/\\/\\\\/g; 1150 1151 do_log("DEBUG DB: $cmd") if $g{'debug'}; 1152 my $result = $g{'dbh'}->do("$cmd") or 1153 do_log("DB write '$cmd' failed; reconnecting",0) 1154 and db_connect() 1155 and return db_do($cmd); 1156 1157 return $result; 1158 } 1159 1160 1161 1162 # Reset owners 1163 sub reset_ownerships { 1164 1165 log_fatal("--initialized only valid when multinode='YES'",0) 1166 if $g{'multinode'} ne 'yes'; 1167 1168 db_connect(); 1169 db_do('update devices set owner=0'); 1170 db_do('update nodes set heartbeat=4294967295,need_tests=0 '. 1171 'where active="y"'); 1172 db_do('delete from test_data'); 1173 1174 die "Database ownerships reset. Please run all active nodes.\n\n"; 1175 1176 } 1177 1178 1179 1180 # Sync the global config on this node to the global config in the db 1181 sub sync_global_config { 1182 1183 # Make sure we are in multinode mode 1184 die "--syncglobal flag on applies if you have the local 'MULTINODE' " . 1185 "option set to 'YES'\n" if $g{'multinode'} ne 'yes'; 1186 1187 # Connect to db 1188 db_connect(); 1189 1190 # Read in our config file 1191 read_global_config_file(); 1192 1193 do_log("Updating global config",0); 1194 # Clear our global config 1195 db_do("delete from global_config"); 1196 1197 # Now go through our options and write them to the DB 1198 for my $opt (sort keys %{$g{'globals'}}) { 1199 my $val = $g{$opt}; 1200 db_do("insert into global_config values ('$opt','$val')"); 1201 } 1202 1203 do_log("Done",0); 1204 1205 # Now quit 1206 &quit(0); 1207 } 1208 1209 1210 # Read in from the bb-hosts file, snmp query hosts to discover their 1211 # vendor and model type, then add them to the DB 1212 sub read_bb_hosts { 1213 my %bb_hosts; 1214 my %new_hosts; 1215 my $sysdesc_oid = '1.3.6.1.2.1.1.1.0'; 1216 my $custom_cids = 0; 1217 my $hosts_left = 0; 1218 1219 # Hashes containing textual shortcuts for bb exception & thresholds 1220 my %thr_sc = ( 'r' => 'red', 'y' => 'yellow', 'g' => 'green' ); 1221 my %exc_sc = ( 'i' => 'ignore', 'o' => 'only', 'ao' => 'alarm', 1222 'na' => 'noalarm' ); 1223 1224 # Read in templates, cause we'll need them 1225 db_connect(); 1226 dm_templates::read_templates(); 1227 1228 # Spew some debug info 1229 if($g{'debug'}) { 1230 my $num_vendor = 0; my $num_model = 0; 1231 my $num_temps = 0; my $num_descs = 0; 1232 for my $vendor (keys %{$g{'templates'}}) { 1233 ++$num_vendor; 1234 for my $model (keys %{$g{'templates'}{$vendor}}) { 1235 ++$num_model; 1236 my $desc = $g{'templates'}{$vendor}{$model}{'sysdesc'}; 1237 $num_descs++ if defined $desc and $desc ne ''; 1238 $num_temps += scalar keys %{$g{'templates'}{$vendor}{$model}}; 1239 } 1240 } 1241 1242 do_log("Saw $num_vendor vendors, $num_model models, " . 1243 "$num_descs sysdescs & $num_temps templates",0); 1244 } 1245 1246 do_log("SNMP querying all hosts in bb-hosts file, please wait...",1); 1247 1248 # Now open the bb-hosts file and read it in 1249 # Also read in any other host files that are included in the bb-hosts 1250 my @bbfiles = ($g{'bbhosts'}); 1251 my $etcdir = $1 if $g{'bbhosts'} =~ /^(.+)\/.+?$/; 1252 $etcdir = $g{'homedir'} if !defined $etcdir; 1253 1254 FILEREAD: do { 1255 1256 my $bbfile = shift @bbfiles; 1257 next if !defined $bbfile; # In case next FILEREAD bypasses the while 1258 1259 # Die if we fail to open our bb root file, warn for all others 1260 if($bbfile eq $g{'bbhosts'}) { 1261 open BBFILE, $bbfile or 1262 log_fatal("Unable to open bb-hosts file '$g{'bbhosts'}' ($!)", 0); 1263 } 1264 else { 1265 open BBFILE, $bbfile or 1266 do_log("Unable to open file '$g{'bbhosts'}' ($!)", 0) and 1267 next FILEREAD; 1268 } 1269 1270 # Now interate through our file and suck out the juicy bits 1271 FILELINE: while ( my $line= <BBFILE> ) { 1272 chomp $line; 1273 1274 while ( $line=~ s/\\$// and ! eof(BBFILE) ) { 1275 $line.= <BBFILE> ; # Merge with next line 1276 chomp $line ; 1277 } # of while 1278 1279 # First see if this is an include statement 1280 if($line =~ /^\s*(?:disp|net)?include\s+(.+)$/i) { 1281 my $file = $1; 1282 # Tack on our etc dir if this isnt an absolute path 1283 $file = "$etcdir/$file" if $file !~ /^\//; 1284 # Add the file to our read array 1285 push @bbfiles, $file; 1286 } 1287 1288 # Else see if this line matches the ip/host bb-hosts format 1289 elsif($line =~ /^\s*(\d+\.\d+\.\d+\.\d+)\s+(\S+)(.*)$/i) { 1290 my ($ip, $host, $bbopts) = ($1, $2, $3); 1291 1292 # Skip if the NET tag does not match this site 1293 do_log("Checking if $bbopts matches NET:" . $g{'bblocation'} . ".",2); 1294 if ($g{'bblocation'} ne '') { 1295 if ($bbopts !~ / NET:$g{'bblocation'}/) { 1296 do_log("The NET for $host is not $g{'bblocation'}. Skipping.",1); 1297 next; 1298 } 1299 } 1300 1301 # See if we can find our bbtag to let us know this is a devmon host 1302 if($bbopts =~ /$g{'bbtag'}(:\S+|\s+|$)/) { 1303 my $options = $1; 1304 $options = '' if !defined $options or $options =~ /^\s+$/; 1305 $options =~ s/,\s+/,/; # Remove spaces in a comma-delimited list 1306 $options =~ s/^://; 1307 1308 # Skip the .default. host, defined 1309 do_log("Can't use Devmon on the .default. host, sorry.",0) 1310 and next if $host eq '.default.'; 1311 1312 # If this IP is 0.0.0.0, try and get IP from DNS 1313 if($ip eq '0.0.0.0') { 1314 my (undef, undef, undef, undef, @addrs) = gethostbyname $host; 1315 do_log("Unable to resolve DNS name for host '$host'",0) 1316 and next FILELINE if !@addrs; 1317 $ip = join '.', unpack('C4', $addrs[0]); 1318 } 1319 1320 # Make sure we dont have duplicates 1321 if(defined $bb_hosts{$host}) { 1322 my $old = $bb_hosts{$host}{'ip'}; 1323 do_log("Refusing to redefine $host from '$old' to '$ip'",0); 1324 next; 1325 } 1326 1327 # See if we have a custom cid 1328 if($options =~ s/(?:,|^)cid\((\S+?)\),?//) { 1329 $bb_hosts{$host}{'cid'} = $1; 1330 $custom_cids = 1; 1331 } 1332 1333 # See if we have a custom port 1334 if($options =~ s/(?:,|^)port\((\d+?)\),?//) { 1335 $bb_hosts{$host}{'port'} = $1; 1336 } 1337 1338 # Look for vendor/model override 1339 if($options =~ s/(?:,|^)model\((\S+?)\),?//) { 1340 my ($vendor, $model) = split /;/, $1, 2; 1341 do_log("Syntax error in model() option for $host",0) and next 1342 if !defined $vendor or !defined $model; 1343 do_log("Unknown vendor in model() option for $host",0) and next 1344 if !defined $g{'templates'}{$vendor}; 1345 do_log("Unknown model in model() option for $host",0) and next 1346 if !defined $g{'templates'}{$vendor}{$model}; 1347 $bb_hosts{$host}{'vendor'} = $vendor; 1348 $bb_hosts{$host}{'model'} = $model; 1349 } 1350 1351 # Read custom exceptions 1352 if($options =~ s/(?:,|^)except\((\S+?)\)//) { 1353 for my $except (split /,/, $1) { 1354 my @args = split /;/, $except; 1355 do_log("Invalid exception clause for $host",0) and next 1356 if scalar @args < 3; 1357 my $test = shift @args; 1358 my $oid = shift @args; 1359 for my $valpair (@args) { 1360 my ($sc, $val) = split /:/, $valpair, 2; 1361 my $type = $exc_sc{$sc}; # Process shortcut text 1362 do_log("Unknown exception shortcut '$sc' for $host") and next 1363 if !defined $type; 1364 $bb_hosts{$host}{'except'}{$test}{$oid}{$type} = $val; 1365 } 1366 } 1367 } 1368 1369 # Read custom thresholds 1370 if($options =~ s/(?:,|^)thresh\((\S+?)\)//) { 1371 for my $thresh (split /,/, $1) { 1372 my @args = split /;/, $thresh; 1373 do_log("Invalid threshold clause for $host",0) and next 1374 if scalar @args < 3; 1375 my $test = shift @args; 1376 my $oid = shift @args; 1377 for my $valpair (@args) { 1378 my ($sc, $val) = split /:/, $valpair, 2; 1379 my $type = $thr_sc{$sc}; # Process shortcut text 1380 do_log("Unknown exception shortcut '$sc' for $host") and next 1381 if !defined $type; 1382 $bb_hosts{$host}{'thresh'}{$test}{$oid}{$type} = $val; 1383 } 1384 } 1385 } 1386 1387 # Default to all tests if they arent defined 1388 my $tests = $1 if $options =~ s/(?:,|^)tests\((\S+?)\)//; 1389 $tests = 'all' if !defined $tests; 1390 1391 do_log("Unknown devmon option ($options) on line " . 1392 "$. of $bbfile",0) and next if $options ne ''; 1393 1394 $bb_hosts{$host}{'ip'} = $ip; 1395 $bb_hosts{$host}{'tests'} = $tests; 1396 1397 # Incremement our host counter, used to tell if we should bother 1398 # trying to query for new hosts... 1399 ++$hosts_left; 1400 } 1401 } 1402 } 1403 close BBFILE; 1404 1405 } while @bbfiles; # End of do {} loop 1406 1407 # Gather our existing hosts 1408 my %old_hosts = read_hosts(); 1409 1410 # Put together our query hash 1411 my %snmp_input; 1412 1413 # Get snmp query params from global conf 1414 read_global_config(); 1415 1416 # First go through our existing hosts and see if they answer snmp 1417 do_log("Querying pre-existing hosts",1) if %old_hosts; 1418 1419 for my $host (keys %old_hosts) { 1420 # If they dont exist in the new bbhosts, skip 'em 1421 next if !defined $bb_hosts{$host}; 1422 1423 my $vendor = $old_hosts{$host}{'vendor'}; 1424 my $model = $old_hosts{$host}{'model'}; 1425 1426 # If their template doesnt exist any more, skip 'em 1427 next if !defined $g{'templates'}{$vendor}{$model}; 1428 1429 my $snmpver = $g{'templates'}{$vendor}{$model}{'snmpver'}; 1430 $snmp_input{$host}{'dev_ip'} = $bb_hosts{$host}{'ip'}; 1431 $snmp_input{$host}{'cid'} = $old_hosts{$host}{'cid'}; 1432 $snmp_input{$host}{'port'} = $old_hosts{$host}{'port'}; 1433 $snmp_input{$host}{'dev'} = $host; 1434 $snmp_input{$host}{'ver'} = $snmpver; 1435 1436 # Add our sysdesc oid 1437 $snmp_input{$host}{'nonreps'}{$sysdesc_oid} = 1; 1438 } 1439 1440 # Throw data to our query forks 1441 dm_snmp::snmp_query(\%snmp_input); 1442 1443 # Now go through our resulting snmp-data 1444 OLDHOST: for my $host (keys %{$g{'snmp_data'}}) { 1445 my $sysdesc = $g{'snmp_data'}{$host}{$sysdesc_oid}{'val'}; 1446 $sysdesc = 'UNDEFINED' if !defined $sysdesc; 1447 do_log("$host sysdesc = ::: $sysdesc :::",0) if $g{'debug'}; 1448 next if $sysdesc eq 'UNDEFINED'; 1449 1450 # Catch vendor/models override with the model() option 1451 if(defined $bb_hosts{$host}{'vendor'}) { 1452 %{$new_hosts{$host}} = %{$bb_hosts{$host}}; 1453 $new_hosts{$host}{'cid'} = $old_hosts{$host}{'cid'}; 1454 $new_hosts{$host}{'port'} = $old_hosts{$host}{'port'}; 1455 1456 --$hosts_left; 1457 do_log("Discovered $host as a $bb_hosts{$host}{'vendor'} " . 1458 "$bb_hosts{$host}{'model'}",2); 1459 next OLDHOST; 1460 } 1461 1462 # Okay, we have a sysdesc, lets see if it matches any of our templates 1463 OLDMATCH: for my $vendor (keys %{$g{'templates'}}) { 1464 OLDMODEL: for my $model (keys %{$g{'templates'}{$vendor}}) { 1465 my $regex = $g{'templates'}{$vendor}{$model}{'sysdesc'}; 1466 1467 # Careful /w those empty regexs 1468 do_log("Regex for $vendor/$model appears to be empty.",0) 1469 and next if !defined $regex; 1470 1471 # Skip if this host doesnt match the regex 1472 if ($sysdesc !~ /$regex/) { 1473 do_log("$host did not match $vendor : $model : $regex", 0) 1474 if $g{'debug'}; 1475 next OLDMODEL; 1476 } 1477 1478 # We got a match, assign the pertinent data 1479 %{$new_hosts{$host}} = %{$bb_hosts{$host}}; 1480 $new_hosts{$host}{'cid'} = $old_hosts{$host}{'cid'}; 1481 $new_hosts{$host}{'port'} = $old_hosts{$host}{'port'}; 1482 $new_hosts{$host}{'vendor'} = $vendor; 1483 $new_hosts{$host}{'model'} = $model; 1484 1485 --$hosts_left; 1486 do_log("Discovered $host as a $vendor $model",2); 1487 last OLDMATCH; 1488 } 1489 } 1490 } 1491 1492 # Now go through each cid from most common to least 1493 my @snmpvers = (2, 1); 1494 1495 # For our new hosts, query them first with snmp v2, then v1 if v2 fails 1496 for my $snmpver (@snmpvers) { 1497 1498 # Dont bother if we dont have any hosts left to query 1499 next if $hosts_left < 1; 1500 1501 # First query hosts with custom cids 1502 if($custom_cids) { 1503 do_log("Querying new hosts /w custom cids using snmp v$snmpver",1); 1504 1505 # Zero out our data in and data out hashes 1506 %{$g{'snmp_data'}} = (); 1507 %snmp_input = (); 1508 1509 for my $host (sort keys %bb_hosts) { 1510 # Skip if they dont have a custom cid 1511 next if !defined $bb_hosts{$host}{'cid'}; 1512 # Skip if they have already been succesfully queried 1513 next if defined $new_hosts{$host}; 1514 1515 # Throw together our query data 1516 $snmp_input{$host}{'dev_ip'} = $bb_hosts{$host}{'ip'}; 1517 $snmp_input{$host}{'cid'} = $bb_hosts{$host}{'cid'}; 1518 $snmp_input{$host}{'port'} = $bb_hosts{$host}{'port'}; 1519 $snmp_input{$host}{'dev'} = $host; 1520 $snmp_input{$host}{'ver'} = $snmpver; 1521 1522 # Add our sysdesc oid 1523 $snmp_input{$host}{'nonreps'}{$sysdesc_oid} = 1; 1524 } 1525 1526 # Reset our failed hosts 1527 $g{'fail'} = {}; 1528 1529 # Throw data to our query forks 1530 dm_snmp::snmp_query(\%snmp_input); 1531 1532 # Now go through our resulting snmp-data 1533 NEWHOST: for my $host (keys %{$g{'snmp_data'}}) { 1534 my $sysdesc = $g{'snmp_data'}{$host}{$sysdesc_oid}{'val'}; 1535 $sysdesc = 'UNDEFINED' if !defined $sysdesc; 1536 do_log("$host sysdesc = ::: $sysdesc :::",0) if $g{'debug'}; 1537 next if $sysdesc eq 'UNDEFINED'; 1538 1539 # Catch vendor/models override with the model() option 1540 if(defined $bb_hosts{$host}{'vendor'}) { 1541 %{$new_hosts{$host}} = %{$bb_hosts{$host}}; 1542 --$hosts_left; 1543 1544 do_log("Discovered $host as a $bb_hosts{$host}{'vendor'} " . 1545 "$bb_hosts{$host}{'model'}",2); 1546 last NEWHOST; 1547 } 1548 1549 # Try and match sysdesc 1550 NEWMATCH: for my $vendor (keys %{$g{'templates'}}) { 1551 NEWMODEL: for my $model (keys %{$g{'templates'}{$vendor}}) { 1552 1553 1554 # Skip if this host doesnt match the regex 1555 my $regex = $g{'templates'}{$vendor}{$model}{'sysdesc'}; 1556 if ($sysdesc !~ /$regex/) { 1557 do_log("$host did not match $vendor : $model : $regex", 0) if $g{'debug'}; 1558 next NEWMODEL; 1559 } 1560 1561 # We got a match, assign the pertinent data 1562 %{$new_hosts{$host}} = %{$bb_hosts{$host}}; 1563 $new_hosts{$host}{'vendor'} = $vendor; 1564 $new_hosts{$host}{'model'} = $model; 1565 --$hosts_left; 1566 1567 # If they are an old host, they probably changes models... 1568 if(defined $old_hosts{$host}) { 1569 my $old_vendor = $old_hosts{$host}{'vendor'}; 1570 my $old_model = $old_hosts{$host}{'model'}; 1571 if($vendor ne $old_vendor or $model ne $old_model) { 1572 do_log("$host changed from a $old_vendor $old_model " . 1573 "to a $vendor $model",1); 1574 } 1575 } 1576 1577 else { 1578 do_log("Discovered $host as a $vendor $model",1); 1579 } 1580 last NEWMATCH; 1581 } 1582 } 1583 1584 # Make sure we were able to get a match 1585 if(!defined $new_hosts{$host}) { 1586 do_log("No matching templates for device: $host",0); 1587 # Delete the bbhosts key so we dont throw another error later 1588 delete $bb_hosts{$host}; 1589 } 1590 } 1591 } 1592 1593 # Now query hosts without custom cids 1594 for my $cid (split /,/, $g{'snmpcids'}) { 1595 1596 # Dont bother if we dont have any hosts left to query 1597 next if $hosts_left < 1; 1598 1599 do_log("Querying new hosts using cid '$cid' and snmp v$snmpver",1); 1600 1601 # Zero out our data in and data out hashes 1602 %{$g{'snmp_data'}} = (); 1603 %snmp_input = (); 1604 1605 # And query the devices that havent yet responded to previous cids 1606 for my $host (sort keys %bb_hosts) { 1607 1608 # Dont query this host if we already have succesfully done so 1609 next if defined $new_hosts{$host}; 1610 1611 $snmp_input{$host}{'dev_ip'} = $bb_hosts{$host}{'ip'}; 1612 $snmp_input{$host}{'port'} = $bb_hosts{$host}{'port'}; 1613 $snmp_input{$host}{'cid'} = $cid; 1614 $snmp_input{$host}{'dev'} = $host; 1615 $snmp_input{$host}{'ver'} = $snmpver; 1616 1617 # Add our sysdesc oid 1618 $snmp_input{$host}{'nonreps'}{$sysdesc_oid} = 1; 1619 } 1620 1621 # Reset our failed hosts 1622 $g{'fail'} = {}; 1623 1624 # Throw data to our query forks 1625 dm_snmp::snmp_query(\%snmp_input); 1626 1627 # Now go through our resulting snmp-data 1628 CUSTOMHOST: for my $host (keys %{$g{'snmp_data'}}) { 1629 my $sysdesc = $g{'snmp_data'}{$host}{$sysdesc_oid}{'val'}; 1630 $sysdesc = 'UNDEFINED' if !defined $sysdesc; 1631 do_log("$host sysdesc = ::: $sysdesc :::",0) if $g{'debug'}; 1632 next if $sysdesc eq 'UNDEFINED'; 1633 1634 # Catch vendor/models override with the model() option 1635 if(defined $bb_hosts{$host}{'vendor'}) { 1636 %{$new_hosts{$host}} = %{$bb_hosts{$host}}; 1637 $new_hosts{$host}{'cid'} = $cid; 1638 --$hosts_left; 1639 1640 do_log("Discovered $host as a $bb_hosts{$host}{'vendor'} " . 1641 "$bb_hosts{$host}{'model'}",2); 1642 next CUSTOMHOST; 1643 } 1644 1645 # Try and match sysdesc 1646 CUSTOMMATCH: for my $vendor (keys %{$g{'templates'}}) { 1647 CUSTOMMODEL: for my $model (keys %{$g{'templates'}{$vendor}}) { 1648 1649 # Skip if this host doesnt match the regex 1650 my $regex = $g{'templates'}{$vendor}{$model}{'sysdesc'}; 1651 if ($sysdesc !~ /$regex/) { 1652 do_log("$host did not match $vendor : $model : $regex", 0) 1653 if $g{'debug'}; 1654 next CUSTOMMODEL; 1655 } 1656 1657 # We got a match, assign the pertinent data 1658 %{$new_hosts{$host}} = %{$bb_hosts{$host}}; 1659 $new_hosts{$host}{'cid'} = $cid; 1660 $new_hosts{$host}{'vendor'} = $vendor; 1661 $new_hosts{$host}{'model'} = $model; 1662 --$hosts_left; 1663 1664 # If they are an old host, they probably changed models... 1665 if(defined $old_hosts{$host}) { 1666 my $old_vendor = $old_hosts{$host}{'vendor'}; 1667 my $old_model = $old_hosts{$host}{'model'}; 1668 if($vendor ne $old_vendor or $model ne $old_model) { 1669 do_log("$host changed from a $old_vendor $old_model " . 1670 "to a $vendor $model",1); 1671 } 1672 } 1673 1674 else { 1675 do_log("Discovered $host as a $vendor $model",1); 1676 } 1677 last CUSTOMMATCH; 1678 } 1679 } 1680 1681 # Make sure we were able to get a match 1682 if(!defined $new_hosts{$host}) { 1683 do_log("No matching templates for device: $host",0); 1684 # Delete the bbhosts key so we dont throw another error later 1685 delete $bb_hosts{$host}; 1686 } 1687 } 1688 } 1689 } 1690 1691 # Go through our bb-hosts and see if we failed any queries on the 1692 # devices; if they were previously defined, just leave them be 1693 # at let them go clear. If they are new, drop a log message 1694 for my $host (keys %bb_hosts) { 1695 next if defined $new_hosts{$host}; 1696 1697 if(defined $old_hosts{$host}) { 1698 # Couldnt query pre-existing host, maybe temporarily unresponsive? 1699 %{$new_hosts{$host}} = %{$old_hosts{$host}}; 1700 } 1701 else { 1702 # Throw a log message complaining 1703 do_log("Could not query device: $host",0); 1704 } 1705 } 1706 1707 # All done, now we just need to write our hosts to the DB 1708 if($g{'multinode'} eq 'yes') { 1709 1710 do_log("Updating database",1); 1711 # Update database 1712 for my $host (keys %new_hosts) { 1713 my $ip = $new_hosts{$host}{'ip'}; 1714 my $vendor = $new_hosts{$host}{'vendor'}; 1715 my $model = $new_hosts{$host}{'model'}; 1716 my $tests = $new_hosts{$host}{'tests'}; 1717 my $cid = $new_hosts{$host}{'cid'}; 1718 my $port = $new_hosts{$host}{'port'}; 1719 1720 $cid .= "::$port" if defined $port; 1721 1722 # Update any pre-existing hosts 1723 if(defined $old_hosts{$host}) { 1724 my $changes = ''; 1725 $changes .= "ip='$ip'," if $ip ne $old_hosts{$host}{'ip'}; 1726 $changes .= "vendor='$vendor'," 1727 if $vendor ne $old_hosts{$host}{'vendor'}; 1728 $changes .= "model='$model'," 1729 if $model ne $old_hosts{$host}{'model'}; 1730 $changes .= "tests='$tests'," 1731 if $tests ne $old_hosts{$host}{'tests'}; 1732 $changes .= "cid='$cid'," if $cid ne $old_hosts{$host}{'cid'}; 1733 1734 # Only update if something changed 1735 if($changes ne '') { 1736 chop $changes; 1737 db_do("update devices set $changes where name='$host'"); 1738 } 1739 1740 # Go through our custom threshes and exceptions, update as needed 1741 for my $test (keys %{$new_hosts{$host}{'thresh'}}) { 1742 for my $oid (keys %{$new_hosts{$host}{'thresh'}{$test}}) { 1743 for my $color (keys %{$new_hosts{$host}{'thresh'}{$test}{$oid}}) { 1744 my $val = $new_hosts{$host}{'thresh'}{$test}{$oid}{$color}; 1745 my $old_val = $old_hosts{$host}{'thresh'}{$test}{$oid}{$color}; 1746 1747 if (defined $val and defined $old_val and $val ne $old_val) { 1748 db_do("update custom_threshs set val='$val' where " . 1749 "host='$host' and test='$test' and color='$color'"); 1750 } 1751 elsif(defined $val and !defined $old_val) { 1752 db_do("delete from custom_threshs where " . 1753 "host='$host' and test='$test' and color='$color'"); 1754 db_do("insert into custom_threshs values " . 1755 "('$host','$test','$oid','$color','$val')"); 1756 } 1757 elsif(!defined $val and defined $old_val) { 1758 db_do("delete from custom_threshs where " . 1759 "host='$host' and test='$test' and color='$color'"); 1760 } 1761 } 1762 } 1763 } 1764 1765 # Exceptions 1766 for my $test (keys %{$new_hosts{$host}{'except'}}) { 1767 for my $oid (keys %{$new_hosts{$host}{'except'}{$test}}) { 1768 for my $type (keys %{$new_hosts{$host}{'except'}{$test}{$oid}}) { 1769 my $val = $new_hosts{$host}{'except'}{$test}{$oid}{$type}; 1770 my $old_val = $old_hosts{$host}{'except'}{$test}{$oid}{$type}; 1771 1772 if (defined $val and defined $old_val and $val ne $old_val) { 1773 db_do("update custom_excepts set data='$val' where " . 1774 "host='$host' and test='$test' and type='$type'"); 1775 } 1776 elsif(defined $val and !defined $old_val) { 1777 db_do("delete from custom_excepts where " . 1778 "host='$host' and test='$test' and type='$type'"); 1779 db_do("insert into custom_excepts values " . 1780 "('$host','$test','$oid','$type','$val')"); 1781 } 1782 elsif(!defined $val and defined $old_val) { 1783 db_do("delete from custom_excepts where " . 1784 "host='$host' and test='$test' and type='$type'"); 1785 } 1786 } 1787 } 1788 } 1789 } 1790 1791 # If it wasnt pre-existing, go ahead and insert it 1792 else { 1793 db_do("delete from devices where name='$host'"); 1794 db_do("insert into devices values ('$host','$ip','$vendor'," . 1795 "'$model','$tests','$cid',0)"); 1796 1797 # Insert new thresholds 1798 for my $test (keys %{$new_hosts{$host}{'thresh'}}) { 1799 for my $oid (keys %{$new_hosts{$host}{'thresh'}{$test}}) { 1800 for my $color (keys %{$new_hosts{$host}{'thresh'}{$test}{$oid}}) { 1801 my $val = $new_hosts{$host}{'thresh'}{$test}{$oid}{$color}; 1802 db_do("insert into custom_threshs values " . 1803 "('$host','$test','$oid','$color','$val')"); 1804 } 1805 } 1806 } 1807 1808 # Insert new exceptions 1809 for my $test (keys %{$new_hosts{$host}{'except'}}) { 1810 for my $oid (keys %{$new_hosts{$host}{'except'}{$test}}) { 1811 for my $type (keys %{$new_hosts{$host}{'except'}{$test}{$oid}}) { 1812 my $val = $new_hosts{$host}{'except'}{$test}{$oid}{$type}; 1813 db_do("insert into custom_excepts values " . 1814 "('$host','$test','$oid','$type','$val')"); 1815 } 1816 } 1817 } 1818 } 1819 } 1820 1821 # Delete any hosts not in the bb hosts file 1822 for my $host (keys %old_hosts) { 1823 next if defined $new_hosts{$host}; 1824 do_log("Removing stale host '$host' from DB",2); 1825 db_do("delete from devices where name='$host'"); 1826 db_do("delete from custom_threshs where host='$host'"); 1827 db_do("delete from custom_excepts where host='$host'"); 1828 } 1829 } 1830 1831 # Or write it to our dbfile if we arent in multinode mode 1832 else { 1833 1834 # Textual abbreviations 1835 my %thr_sc = ( 'red' => 'r', 'yellow' => 'y', 'green' => 'g' ); 1836 my %exc_sc = ( 'ignore' => 'i', 'only' => 'o', 'alarm' => 'ao', 1837 'noalarm' => 'na' ); 1838 open HOSTFILE, ">$g{'dbfile'}" 1839 or log_fatal("Unable to write to dbfile '$g{'dbfile'}' ($!)",0); 1840 1841 for my $host (sort keys %new_hosts) { 1842 my $ip = $new_hosts{$host}{'ip'}; 1843 my $vendor = $new_hosts{$host}{'vendor'}; 1844 my $model = $new_hosts{$host}{'model'}; 1845 my $tests = $new_hosts{$host}{'tests'}; 1846 my $cid = $new_hosts{$host}{'cid'}; 1847 my $port = $new_hosts{$host}{'port'}; 1848 1849 $cid .= "::$port" if defined $port; 1850 1851 # Custom thresholds 1852 my $threshes = ''; 1853 for my $test (keys %{$new_hosts{$host}{'thresh'}}) { 1854 for my $oid (keys %{$new_hosts{$host}{'thresh'}{$test}}) { 1855 $threshes .= "$test;$oid"; 1856 for my $color (keys %{$new_hosts{$host}{'thresh'}{$test}{$oid}}) { 1857 my $val = $new_hosts{$host}{'thresh'}{$test}{$oid}{$color}; 1858 my $sc = $thr_sc{$color}; 1859 $threshes .= ";$sc:$val"; 1860 } 1861 $threshes .= ','; 1862 } 1863 $threshes .= ',' if ($threshes !~ /,$/); 1864 } 1865 $threshes =~ s/,$//; 1866 1867 # Custom exceptions 1868 my $excepts = ''; 1869 for my $test (keys %{$new_hosts{$host}{'except'}}) { 1870 for my $oid (keys %{$new_hosts{$host}{'except'}{$test}}) { 1871 $excepts .= "$test;$oid"; 1872 for my $type (keys %{$new_hosts{$host}{'except'}{$test}{$oid}}) { 1873 my $val = $new_hosts{$host}{'except'}{$test}{$oid}{$type}; 1874 my $sc = $exc_sc{$type}; 1875 $excepts .= ";$sc:$val"; 1876 } 1877 $excepts .= ','; 1878 } 1879 $excepts .= ',' if ($excepts !~ /,$/); 1880 } 1881 $excepts =~ s/,$//; 1882 1883 print HOSTFILE "$host\e$ip\e$vendor\e$model\e$tests\e$cid\e" . 1884 "$threshes\e$excepts\n"; 1885 } 1886 1887 close HOSTFILE; 1888 } 1889 1890 # Now quit 1891 &quit(0); 1892 } 1893 1894 1895 1896 # Read hosts in from mysql DB in multinode mode, or else from disk 1897 sub read_hosts { 1898 1899 my %hosts = (); 1900 1901 do_log("DEBUG CFG: running read_hosts",0) if $g{'debug'}; 1902 1903 # Multinode 1904 if($g{'multinode'} eq 'yes') { 1905 my @arr = db_get_array("name,ip,vendor,model,tests,cid from devices"); 1906 for my $host (@arr) { 1907 my ($name,$ip,$vendor,$model,$tests,$cid) = @$host; 1908 1909 my $port = $1 if $cid =~ s/::(\d+)$//; 1910 1911 $hosts{$name}{'ip'} = $ip; 1912 $hosts{$name}{'vendor'} = $vendor; 1913 $hosts{$name}{'model'} = $model; 1914 $hosts{$name}{'tests'} = $tests; 1915 $hosts{$name}{'cid'} = $cid; 1916 $hosts{$name}{'port'} = $port; 1917 } 1918 1919 @arr = db_get_array("host,test,oid,type,data from custom_excepts"); 1920 for my $except (@arr) { 1921 my ($name,$test,$oid,$type,$data) = @$except; 1922 $hosts{$name}{'except'}{$test}{$oid}{$type} = $data 1923 if defined $hosts{$name}; 1924 } 1925 1926 @arr = db_get_array("host,test,oid,color,val from custom_threshs"); 1927 for my $thresh (@arr) { 1928 my ($name,$test,$oid,$color,$val) = @$thresh; 1929 $hosts{$name}{'thresh'}{$test}{$oid}{$color} = $val 1930 if defined $hosts{$name}; 1931 } 1932 } 1933 1934 # Singlenode 1935 else { 1936 1937 # Hashes containing textual shortcuts for bb exception & thresholds 1938 my %thr_sc = ( 'r' => 'red', 'y' => 'yellow', 'g' => 'green' ); 1939 my %exc_sc = ( 'i' => 'ignore', 'o' => 'only', 'ao' => 'alarm', 1940 'na' => 'noalarm' ); 1941 # Statistic variables (done here in singlenode, instead of syncservers) 1942 my $numdevs; 1943 my $numtests; 1944 1945 # Check if the hosts file even exists 1946 return %hosts if !-e $g{'dbfile'}; 1947 1948 # Open and read in data 1949 open HOSTS, $g{'dbfile'} or 1950 log_fatal("Unable to open host file: $g{'dbfile'} ($!)", 0); 1951 1952 my $num; 1953 FILELINE: for my $line (<HOSTS>) { 1954 chomp $line; 1955 my ($name,$ip,$vendor,$model,$tests,$cid,$threshes,$excepts) 1956 = split /\e/, $line; 1957 ++$num; 1958 1959 do_log("Invalid entry in host file at line $num.",0) and next 1960 if !defined $cid; 1961 1962 my $port = $1 if $cid =~ s/::(\d+)$//; 1963 1964 $hosts{$name}{'ip'} = $ip; 1965 $hosts{$name}{'vendor'} = $vendor; 1966 $hosts{$name}{'model'} = $model; 1967 $hosts{$name}{'tests'} = $tests; 1968 $hosts{$name}{'cid'} = $cid; 1969 $hosts{$name}{'port'} = $port; 1970 1971 if(defined $threshes and $threshes ne '') { 1972 for my $thresh (split ',', $threshes) { 1973 my @args = split /;/, $thresh, 4; 1974 my $test = shift @args; 1975 my $oid = shift @args; 1976 for my $valpair (@args) { 1977 my ($sc, $val) = split /:/, $valpair, 2; 1978 my $color = $thr_sc{$sc}; 1979 $hosts{$name}{'thresh'}{$test}{$oid}{$color} = $val; 1980 } 1981 } 1982 } 1983 1984 if(defined $excepts and $excepts ne '') { 1985 for my $except (split ',', $excepts) { 1986 my @args = split /;/, $except, 4; 1987 my $test = shift @args; 1988 my $oid = shift @args; 1989 for my $valpair (@args) { 1990 my ($sc, $val) = split /:/, $valpair, 2; 1991 my $type = $exc_sc{$sc}; 1992 $hosts{$name}{'except'}{$test}{$oid}{$type} = $val; 1993 } 1994 } 1995 } 1996 1997 # Statistics 1998 ++$numdevs; 1999 $numtests += ($tests =~ tr/,/,/) + 1; 2000 } 2001 close HOSTS; 2002 2003 $g{'numdevs'} = $numdevs; 2004 $g{'numtests'} = $numtests; 2005 $g{'avgtestsnode'} = 'n/a'; 2006 } 2007 2008 return %hosts; 2009 2010 } 2011 2012 2013 # Daemonize: go to daemon mode and fork into background 2014 # Much code shamelessly stolen from Proc::Daemon by Earl Hood 2015 sub daemonize { 2016 2017 return if !$g{'daemonize'}; 2018 2019 # Now fork our child process off 2020 if(my $pid = do_fork()) { 2021 # Parent process, we should die 2022 do_log("Forking to background process $pid",1); 2023 exit 0; 2024 } 2025 2026 # Child process; make sure we disconnect from TTY completely 2027 POSIX::setsid(); 2028 2029 # Prevent possibility of acquiring a controling terminal 2030 $SIG{'HUP'} = 'IGNORE'; 2031 exit 0 if do_fork(); 2032 2033 # Clear file creation mask 2034 umask 0; 2035 2036 # Close open file descriptors 2037 my $openmax = POSIX::sysconf( &POSIX::_SC_OPEN_MAX ); 2038 $openmax = 64 if !defined $openmax or $openmax < 0; 2039 for my $i (0 .. $openmax) { POSIX::close($i) } 2040 2041 # Reopen stderr, stdout, stdin to /dev/null 2042 open(STDIN, "+>/dev/null"); 2043 open(STDOUT, "+>&STDIN"); 2044 open(STDERR, "+>&STDIN"); 2045 2046 # Define ourselves as the master 2047 $0 = 'devmon[master]'; 2048 2049 # Set up our signal handlers again, just to be sure 2050 $SIG{INT} = $SIG{QUIT} = $SIG{TERM} = \&quit; 2051 $SIG{HUP} = \&reopen_log; 2052 2053 } 2054 2055 2056 2057 # Fork with retries. 2058 sub do_fork { 2059 my ($pid, $tries); 2060 FORK: { 2061 if (defined($pid = fork)) { 2062 return $pid; 2063 } 2064 2065 # If we are out of process space, wait 1 second, then try 4 more times 2066 elsif ($! =~ /No more process/ and ++$tries < 5) { 2067 sleep 1; 2068 redo FORK; 2069 } 2070 elsif($! ne '') { 2071 log_fatal("Can't fork: $!",0); 2072 } 2073 } 2074 } 2075 2076 2077 2078 # Find path to a binary file on the system 2079 sub bin_path { 2080 my ($bin) = @_; 2081 2082 # Determine where we should search for binaries 2083 my @pathdirs; 2084 @pathdirs = split /:/, $ENV{'PATH'} if defined $ENV{'PATH'}; 2085 @pathdirs = ('/bin','/usr/bin','/usr/local/bin') if $#pathdirs == -1; 2086 2087 # Now iterate through our dirs, and return if we find a binary 2088 for my $dir (@pathdirs) { 2089 2090 # Remove any trailing slashes 2091 $dir =~ s/(.+)\/$/$1/; 2092 return "$dir/$bin" if -x "$dir/$bin"; 2093 } 2094 2095 # Didnt find it, return undef 2096 return undef; 2097 } 2098 2099 2100 2101 # Sub called by sort, returns results numerically ascending 2102 sub na { $a <=> $b } 2103 2104 2105 2106 # Sub called by sort, returns results numerically descending 2107 sub nd { $b <=> $a } 2108 2109 2110 # Print help 2111 sub usage { 2112 die 2113 "Devmon v$g{'version'}, a device monitor for BigBrother/Hobbit\n" . 2114 "\n" . 2115 "Usage: devmon [arguments]\n" . 2116 "\n" . 2117 " Arguments:\n" . 2118 " -c Specify config file location\n" . 2119 " -d Specify database file location\n" . 2120 " -f Run in foreground. Prevents running in daemon mode.\n" . 2121 " -p Print message. Don't send message to display server.\n" . 2122 " print it to stdout\n" . 2123 " -v Verbose mode. The more v's, the more vebose logging.\n" . 2124 " --debug Print debug output (this can be quite extensive).\n" . 2125 "\n" . 2126 " Mutually exclusive arguments:\n" . 2127 " --readbbhosts Read in data from the BigBrother/Hobbit hosts file\n" . 2128 " --syncconfig Update multinode DB with the global config options\n" . 2129 " configured on this local node.\n" . 2130 " --synctemplates Update multinode device templates with the template\n" . 2131 " data on this local node.\n" . 2132 " --resetowners Reset multinode device ownership data. This will\n" . 2133 " cause all nodes to recalculate ownership data.\n" . 2134 "\n"; 2135 } 2136 2137 # Sub to call when we quit, be it normally or not 2138 sub quit { 2139 my ($retcode) = @_; 2140 $retcode = 0 if (!defined $retcode); 2141 if ($retcode !~ /^\d*$/) { 2142 do_log("Received signal $retcode, shutting down with return code 0",3); 2143 $retcode = 0; 2144 } 2145 2146 $g{'shutting_down'} = 1; 2147 2148 # Only run this if we are the parent process 2149 if($g{'parent'}) { 2150 do_log("Shutting down",0) if $g{'initialized'}; 2151 unlink $g{'pidfile'} if $g{'initialized'} and -e $g{'pidfile'}; 2152 $g{'log'}->close if defined $g{'log'} and $g{'log'} ne ''; 2153 $g{'dbh'}->disconnect() if defined $g{'dbh'} and $g{'dbh'} ne ''; 2154 2155 # Clean up our forks if we left any behind, first by killing them nicely 2156 for my $fork (keys %{$g{'forks'}}) { 2157 my $pid = $g{'forks'}{$fork}{'pid'}; 2158 kill 15, $pid if defined $pid; 2159 } 2160 sleep 1; 2161 # Then, if they are still hanging around... 2162 for my $fork (keys %{$g{'forks'}}) { 2163 my $pid = $g{'forks'}{$fork}{'pid'}; 2164 kill 9, $pid if defined $pid and kill 0, $pid; # Kick their asses 2165 } 2166 2167 } 2168 2169 exit $retcode; 2170 } 2171 2172 END { 2173 &quit if !$g{'shutting_down'}; 2174 } 2175 2176 21771; 2178 2179