1#!/usr/local/bin/python3.8 2# Copyright (C) 2017 Xin Liang <XLiang@suse.com> 3# See COPYING for license information. 4 5import getopt 6import multiprocessing 7import os 8import re 9import sys 10import datetime 11import shutil 12 13sys.path.append(os.path.dirname(os.path.realpath(__file__))) 14import constants 15import utillib 16from crmsh import utils as crmutils 17from crmsh import config 18 19def collect_for_nodes(nodes, arg_str): 20 """ 21 Start slave collectors 22 """ 23 for node in nodes.split(): 24 if utillib.node_needs_pwd(node): 25 utillib.log_info("Please provide password for %s at %s" % (utillib.say_ssh_user(), node)) 26 utillib.log_info("Note that collecting data will take a while.") 27 utillib.start_slave_collector(node, arg_str) 28 else: 29 p = multiprocessing.Process(target=utillib.start_slave_collector, args=(node, arg_str)) 30 p.start() 31 p.join() 32 33def dump_env(): 34 """ 35 this is how we pass environment to other hosts 36 """ 37 env_dict = {} 38 env_dict["DEST"] = constants.DEST 39 env_dict["FROM_TIME"] = constants.FROM_TIME 40 env_dict["TO_TIME"] = constants.TO_TIME 41 env_dict["USER_NODES"] = constants.USER_NODES 42 env_dict["NODES"] = constants.NODES 43 env_dict["HA_LOG"] = constants.HA_LOG 44 # env_dict["UNIQUE_MSG"] = constants.UNIQUE_MSG 45 env_dict["SANITIZE_RULE_DICT"] = constants.SANITIZE_RULE_DICT 46 env_dict["DO_SANITIZE"] = constants.DO_SANITIZE 47 env_dict["SKIP_LVL"] = constants.SKIP_LVL 48 env_dict["EXTRA_LOGS"] = constants.EXTRA_LOGS 49 env_dict["PCMK_LOG"] = constants.PCMK_LOG 50 env_dict["VERBOSITY"] = int(constants.VERBOSITY) 51 52 res_str = "" 53 for k, v in env_dict.items(): 54 res_str += " {}={}".format(k, v) 55 return res_str 56 57def get_log(): 58 """ 59 get the right part of the log 60 """ 61 outf = os.path.join(constants.WORKDIR, constants.HALOG_F) 62 63 # collect journal from systemd unless -M was passed 64 if constants.EXTRA_LOGS: 65 utillib.collect_journal(constants.FROM_TIME, 66 constants.TO_TIME, 67 os.path.join(constants.WORKDIR, constants.JOURNAL_F)) 68 69 if constants.HA_LOG and not os.path.isfile(constants.HA_LOG): 70 if not is_collector(): # warning if not on slave 71 utillib.log_warning("%s not found; we will try to find log ourselves" % constants.HA_LOG) 72 constants.HA_LOG = "" 73 if not constants.HA_LOG: 74 constants.HA_LOG = utillib.find_log() 75 if (not constants.HA_LOG) or (not os.path.isfile(constants.HA_LOG)): 76 if constants.CTS: 77 pass # TODO 78 else: 79 utillib.log_warning("not log at %s" % constants.WE) 80 return 81 82 if constants.CTS: 83 pass # TODO 84 else: 85 try: 86 getstampproc = utillib.find_getstampproc(constants.HA_LOG) 87 except PermissionError: 88 return 89 if getstampproc: 90 constants.GET_STAMP_FUNC = getstampproc 91 if utillib.dump_logset(constants.HA_LOG, constants.FROM_TIME, constants.TO_TIME, outf): 92 utillib.log_size(constants.HA_LOG, outf+'.info') 93 else: 94 utillib.log_warning("could not figure out the log format of %s" % constants.HA_LOG) 95 96 97def is_collector(): 98 """ 99 the instance where user runs hb_report is the master 100 the others are slaves 101 """ 102 if len(sys.argv) > 1 and sys.argv[1] == "__slave": 103 return True 104 return False 105 106 107def load_env(env_str): 108 list_ = [] 109 for tmp in env_str.split(): 110 if re.search('=', tmp): 111 item = tmp 112 else: 113 list_.remove(item) 114 item += " %s" % tmp 115 list_.append(item) 116 117 env_dict = {} 118 env_dict = crmutils.nvpairs2dict(list_) 119 constants.DEST = env_dict["DEST"] 120 constants.FROM_TIME = float(env_dict["FROM_TIME"]) 121 constants.TO_TIME = float(env_dict["TO_TIME"]) 122 constants.USER_NODES = env_dict["USER_NODES"] 123 constants.NODES = env_dict["NODES"] 124 constants.HA_LOG = env_dict["HA_LOG"] 125 # constants.UNIQUE_MSG = env_dict["UNIQUE_MSG"] 126 constants.SANITIZE_RULE_DICT = env_dict["SANITIZE_RULE_DICT"] 127 constants.DO_SANITIZE = env_dict["DO_SANITIZE"] 128 constants.SKIP_LVL = utillib.str_to_bool(env_dict["SKIP_LVL"]) 129 constants.EXTRA_LOGS = env_dict["EXTRA_LOGS"] 130 constants.PCMK_LOG = env_dict["PCMK_LOG"] 131 constants.VERBOSITY = int(env_dict["VERBOSITY"]) 132 133 134def parse_argument(argv): 135 try: 136 opt, arg = getopt.getopt(argv[1:], constants.ARGOPTS_VALUE) 137 except getopt.GetoptError: 138 usage("short") 139 140 if len(arg) == 0: 141 constants.DESTDIR = "." 142 constants.DEST = "hb_report-%s" % datetime.datetime.now().strftime('%a-%d-%b-%Y') 143 elif len(arg) == 1: 144 constants.TMP = arg[0] 145 else: 146 usage("short") 147 148 for args, option in opt: 149 if args == '-h': 150 usage() 151 if args == "-V": 152 version() 153 if args == '-f': 154 constants.FROM_TIME = crmutils.parse_to_timestamp(option) 155 utillib.check_time(constants.FROM_TIME, option) 156 if args == '-t': 157 constants.TO_TIME = crmutils.parse_to_timestamp(option) 158 utillib.check_time(constants.TO_TIME, option) 159 if args == "-n": 160 constants.USER_NODES += " %s" % option 161 if args == "-u": 162 constants.SSH_USER = option 163 if args == "-X": 164 constants.SSH_OPTS += " %s" % option 165 if args == "-l": 166 constants.HA_LOG = option 167 if args == "-e": 168 constants.EDITOR = option 169 if args == "-p": 170 constants.SANITIZE_RULE += " %s" % option 171 if args == "-s": 172 constants.DO_SANITIZE = True 173 if args == "-Q": 174 constants.SKIP_LVL = True 175 if args == "-L": 176 constants.LOG_PATTERNS += " %s" % option 177 if args == "-S": 178 constants.NO_SSH = True 179 if args == "-D": 180 constants.NO_DESCRIPTION = 1 181 if args == "-Z": 182 constants.FORCE_REMOVE_DEST = True 183 if args == "-M": 184 constants.EXTRA_LOGS = "" 185 if args == "-E": 186 constants.EXTRA_LOGS += " %s" % option 187 if args == "-v": 188 constants.VERBOSITY += 1 189 if args == '-d': 190 constants.COMPRESS = False 191 192 if config.report.sanitize_rule: 193 constants.DO_SANITIZE = True 194 temp_pattern_set = set() 195 temp_pattern_set |= set(re.split('\s*\|\s*|\s+', config.report.sanitize_rule.strip('|'))) 196 constants.SANITIZE_RULE += " {}".format(' '.join(temp_pattern_set)) 197 utillib.parse_sanitize_rule(constants.SANITIZE_RULE) 198 199 if not constants.FROM_TIME: 200 from_time = config.report.from_time 201 if re.search("^-[1-9][0-9]*[YmdHM]$", from_time): 202 number = int(re.findall("[1-9][0-9]*", from_time)[0]) 203 if re.search("^-[1-9][0-9]*Y$", from_time): 204 timedelta = datetime.timedelta(days = number * 365) 205 if re.search("^-[1-9][0-9]*m$", from_time): 206 timedelta = datetime.timedelta(days = number * 30) 207 if re.search("^-[1-9][0-9]*d$", from_time): 208 timedelta = datetime.timedelta(days = number) 209 if re.search("^-[1-9][0-9]*H$", from_time): 210 timedelta = datetime.timedelta(hours = number) 211 if re.search("^-[1-9][0-9]*M$", from_time): 212 timedelta = datetime.timedelta(minutes = number) 213 from_time = (datetime.datetime.now() - timedelta).strftime("%Y-%m-%d %H:%M") 214 constants.FROM_TIME = crmutils.parse_to_timestamp(from_time) 215 utillib.check_time(constants.FROM_TIME, from_time) 216 else: 217 utillib.log_fatal("Wrong format for from_time in /etc/crm/crm.conf; (-[1-9][0-9]*[YmdHM])") 218 219 220def run(): 221 222 utillib.check_env() 223 tmpdir = utillib.make_temp_dir() 224 utillib.add_tempfiles(tmpdir) 225 226 # 227 # get and check options; and the destination 228 # 229 if not is_collector(): 230 parse_argument(sys.argv) 231 set_dest(constants.TMP) 232 constants.WORKDIR = os.path.join(tmpdir, constants.DEST) 233 else: 234 constants.WORKDIR = os.path.join(tmpdir, constants.DEST, constants.WE) 235 utillib._mkdir(constants.WORKDIR) 236 237 if is_collector(): 238 load_env(' '.join(sys.argv[2:])) 239 240 utillib.compatibility_pcmk() 241 if constants.CTS == "" or is_collector(): 242 utillib.get_log_vars() 243 244 if not is_collector(): 245 constants.NODES = ' '.join(utillib.get_nodes()) 246 utillib.log_debug("nodes: %s" % constants.NODES) 247 if constants.NODES == "": 248 utillib.log_fatal("could not figure out a list of nodes; is this a cluster node?") 249 if constants.WE in constants.NODES.split(): 250 constants.THIS_IS_NODE = 1 251 252 if not is_collector(): 253 if constants.THIS_IS_NODE != 1: 254 utillib.log_warning("this is not a node and you didn't specify a list of nodes using -n") 255 # 256 # ssh business 257 # 258 if not constants.NO_SSH: 259 # if the ssh user was supplied, consider that it 260 # works; helps reduce the number of ssh invocations 261 utillib.find_ssh_user() 262 if constants.SSH_USER: 263 constants.SSH_OPTS += " -o User=%s" % constants.SSH_USER 264 # assume that only root can collect data 265 if ((not constants.SSH_USER) and (os.getuid() != 0)) or \ 266 constants.SSH_USER and constants.SSH_USER != "root": 267 utillib.log_debug("ssh user other than root, use sudo") 268 constants.SUDO = "sudo -u root" 269 if os.getuid() != 0: 270 utillib.log_debug("local user other than root, use sudo") 271 constants.LOCAL_SUDO = "sudo -u root" 272 273 # 274 # find the logs and cut out the segment for the period 275 # 276 if constants.THIS_IS_NODE == 1: 277 get_log() 278 279 if not is_collector(): 280 arg_str = dump_env() 281 if not constants.NO_SSH: 282 collect_for_nodes(constants.NODES, arg_str) 283 elif constants.THIS_IS_NODE == 1: 284 collect_for_nodes(constants.WE, arg_str) 285 286 # 287 # endgame: 288 # slaves tar their results to stdout, the master waits 289 # for them, analyses results, asks the user to edit the 290 # problem description template, and prints final notes 291 # 292 if is_collector(): 293 utillib.collect_info() 294 cmd = r"cd %s/.. && tar -h -cf - %s" % (constants.WORKDIR, constants.WE) 295 code, out, err = crmutils.get_stdout_stderr(cmd, raw=True) 296 print("{}{}".format(constants.COMPRESS_DATA_FLAG, out)) 297 else: 298 p_list = [] 299 p_list.append(multiprocessing.Process(target=utillib.analyze)) 300 p_list.append(multiprocessing.Process(target=utillib.events, args=(constants.WORKDIR,))) 301 for p in p_list: 302 p.start() 303 304 utillib.check_if_log_is_empty() 305 utillib.mktemplate(sys.argv) 306 307 for p in p_list: 308 p.join() 309 310 if not constants.SKIP_LVL: 311 utillib.sanitize() 312 313 if constants.COMPRESS: 314 utillib.pick_compress() 315 cmd = r"(cd %s/.. && tar cf - %s)|%s > %s/%s.tar%s" % ( 316 constants.WORKDIR, constants.DEST, constants.COMPRESS_PROG, 317 constants.DESTDIR, constants.DEST, constants.COMPRESS_EXT) 318 crmutils.ext_cmd(cmd) 319 else: 320 shutil.move(constants.WORKDIR, constants.DESTDIR) 321 utillib.finalword() 322 323 324def set_dest(dest): 325 """ 326 default DEST has already been set earlier (if the 327 argument is missing) 328 """ 329 if dest: 330 constants.DESTDIR = utillib.get_dirname(dest) 331 constants.DEST = os.path.basename(dest) 332 if not os.path.isdir(constants.DESTDIR): 333 utillib.log_fatal("%s is illegal directory name" % constants.DESTDIR) 334 if not crmutils.is_filename_sane(constants.DEST): 335 utillib.log_fatal("%s contains illegal characters" % constants.DEST) 336 if not constants.COMPRESS and os.path.isdir(os.path.join(constants.DESTDIR, constants.DEST)): 337 if constants.FORCE_REMOVE_DEST: 338 shutil.rmtree(os.path.join(constants.DESTDIR, constants.DEST)) 339 else: 340 utillib.log_fatal("destination directory DESTDIR/DEST exists, please cleanup or use -Z") 341 342 343def usage(short_msg=''): 344 print(""" 345usage: report -f {time} [-t time] 346 [-u user] [-X ssh-options] [-l file] [-n nodes] [-E files] 347 [-p patt] [-L patt] [-e prog] [-MSDZQVsvhd] [dest] 348 349 -f time: time to start from 350 -t time: time to finish at (dflt: now) 351 -d : don't compress, but leave result in a directory 352 -n nodes: node names for this cluster; this option is additive 353 (use either -n "a b" or -n a -n b) 354 if you run report on the loghost or use autojoin, 355 it is highly recommended to set this option 356 -u user: ssh user to access other nodes (dflt: empty, root, hacluster) 357 -X ssh-options: extra ssh(1) options 358 -l file: log file 359 -E file: extra logs to collect; this option is additive 360 (dflt: /var/log/messages) 361 -s : sanitize the PE and CIB files 362 -p patt: regular expression to match variables containing sensitive data; 363 this option is additive (dflt: "passw.*") 364 -L patt: regular expression to match in log files for analysis; 365 this option is additive (dflt: CRIT: ERROR:) 366 -e prog: your favourite editor 367 -Q : don't run resource intensive operations (speed up) 368 -M : don't collect extra logs (/var/log/messages) 369 -D : don't invoke editor to write description 370 -Z : if destination directories exist, remove them instead of exiting 371 (this is default for CTS) 372 -S : single node operation; don't try to start report 373 collectors on other nodes 374 -v : increase verbosity 375 -V : print version 376 dest : report name (may include path where to store the report) 377 """) 378 if short_msg != "short": 379 print(""" 380 . the multifile output is stored in a tarball {dest}.tar.bz2 381 . the time specification is as in either Date::Parse or 382 Date::Manip, whatever you have installed; Date::Parse is 383 preferred 384 . we try to figure where is the logfile; if we can't, please 385 clue us in ('-l') 386 . we collect only one logfile and /var/log/messages; if you 387 have more than one logfile, then use '-E' option to supply 388 as many as you want ('-M' empties the list) 389 390 Examples 391 392 report -f 2pm report_1 393 report -f "2007/9/5 12:30" -t "2007/9/5 14:00" report_2 394 report -f 1:00 -t 3:00 -l /var/log/cluster/ha-debug report_3 395 report -f "09-sep-07 2:00" -u hbadmin report_4 396 report -f 18:00 -p "usern.*" -p "admin.*" report_5 397 398 . WARNING . WARNING . WARNING . WARNING . WARNING . WARNING . 399 400 We won't sanitize the CIB and the peinputs files, because 401 that would make them useless when trying to reproduce the 402 PE behaviour. You may still choose to obliterate sensitive 403 information if you use the -s and -p options, but in that 404 case the support may be lacking as well. The logs and the 405 crm_mon, ccm_tool, and crm_verify output are *not* sanitized. 406 407 Additional system logs (/var/log/messages) are collected in 408 order to have a more complete report. If you don't want that 409 specify -M. 410 411 IT IS YOUR RESPONSIBILITY TO PROTECT THE DATA FROM EXPOSURE! 412 """) 413 sys.exit(1) 414 415 416def version(): 417 print(utillib.crmsh_info().strip('\n')) 418 sys.exit(0) 419 420 421if __name__ == "__main__": 422 try: 423 run() 424 except UnicodeDecodeError: 425 import traceback 426 traceback.print_exc() 427 sys.stdout.flush() 428 429# vim:ts=4:sw=4:et: 430