1# Copyright 2008-2015 Nokia Networks 2# Copyright 2016- Robot Framework Foundation 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16import os 17import random 18import sys 19import time 20 21from robot.errors import DataError, FrameworkError 22from robot.output import LOGGER, loggerhelper 23from robot.result.keywordremover import KeywordRemover 24from robot.result.flattenkeywordmatcher import validate_flatten_keyword 25from robot.utils import (abspath, escape, format_time, get_link_path, 26 html_escape, is_list_like, py2to3, 27 split_args_from_name_or_path) 28 29from .gatherfailed import gather_failed_tests, gather_failed_suites 30 31 32@py2to3 33class _BaseSettings(object): 34 _cli_opts = {'RPA' : ('rpa', None), 35 'Name' : ('name', None), 36 'Doc' : ('doc', None), 37 'Metadata' : ('metadata', []), 38 'TestNames' : ('test', []), 39 'TaskNames' : ('task', []), 40 'ReRunFailed' : ('rerunfailed', 'NONE'), 41 'ReRunFailedSuites': ('rerunfailedsuites', 'NONE'), 42 'SuiteNames' : ('suite', []), 43 'SetTag' : ('settag', []), 44 'Include' : ('include', []), 45 'Exclude' : ('exclude', []), 46 'Critical' : ('critical', None), 47 'NonCritical' : ('noncritical', None), 48 'OutputDir' : ('outputdir', abspath('.')), 49 'Log' : ('log', 'log.html'), 50 'Report' : ('report', 'report.html'), 51 'XUnit' : ('xunit', None), 52 'SplitLog' : ('splitlog', False), 53 'TimestampOutputs' : ('timestampoutputs', False), 54 'LogTitle' : ('logtitle', None), 55 'ReportTitle' : ('reporttitle', None), 56 'ReportBackground' : ('reportbackground', 57 ('#9e9', '#9e9', '#f66')), 58 'SuiteStatLevel' : ('suitestatlevel', -1), 59 'TagStatInclude' : ('tagstatinclude', []), 60 'TagStatExclude' : ('tagstatexclude', []), 61 'TagStatCombine' : ('tagstatcombine', []), 62 'TagDoc' : ('tagdoc', []), 63 'TagStatLink' : ('tagstatlink', []), 64 'RemoveKeywords' : ('removekeywords', []), 65 'FlattenKeywords' : ('flattenkeywords', []), 66 'PreRebotModifiers': ('prerebotmodifier', []), 67 'StatusRC' : ('statusrc', True), 68 'ConsoleColors' : ('consolecolors', 'AUTO'), 69 'StdOut' : ('stdout', None), 70 'StdErr' : ('stderr', None), 71 'XUnitSkipNonCritical' : ('xunitskipnoncritical', False)} 72 _output_opts = ['Output', 'Log', 'Report', 'XUnit', 'DebugFile'] 73 74 def __init__(self, options=None, **extra_options): 75 self.start_timestamp = format_time(time.time(), '', '-', '') 76 self._opts = {} 77 self._cli_opts = self._cli_opts.copy() 78 self._cli_opts.update(self._extra_cli_opts) 79 self._process_cli_opts(dict(options or {}, **extra_options)) 80 81 def _process_cli_opts(self, opts): 82 for name, (cli_name, default) in self._cli_opts.items(): 83 value = opts[cli_name] if cli_name in opts else default 84 if isinstance(default, list): 85 # Copy mutable values and support list values as scalars. 86 value = list(value) if is_list_like(value) else [value] 87 self[name] = self._process_value(name, value) 88 self['TestNames'] += self['ReRunFailed'] + self['TaskNames'] 89 self['SuiteNames'] += self['ReRunFailedSuites'] 90 91 def __setitem__(self, name, value): 92 if name not in self._cli_opts: 93 raise KeyError("Non-existing option '%s'." % name) 94 self._opts[name] = value 95 96 def _process_value(self, name, value): 97 if name == 'ReRunFailed': 98 return gather_failed_tests(value) 99 if name == 'ReRunFailedSuites': 100 return gather_failed_suites(value) 101 if name == 'LogLevel': 102 return self._process_log_level(value) 103 if value == self._get_default_value(name): 104 return value 105 if name == 'Doc': 106 return self._escape_as_data(value) 107 if name in ['Metadata', 'TagDoc']: 108 if name == 'Metadata': 109 value = [self._escape_as_data(v) for v in value] 110 return [self._process_metadata_or_tagdoc(v) for v in value] 111 if name in ['Include', 'Exclude']: 112 return [self._format_tag_patterns(v) for v in value] 113 if name in self._output_opts and (not value or value.upper() == 'NONE'): 114 return None 115 if name == 'OutputDir': 116 return abspath(value) 117 if name in ['SuiteStatLevel', 'ConsoleWidth']: 118 return self._convert_to_positive_integer_or_default(name, value) 119 if name == 'VariableFiles': 120 return [split_args_from_name_or_path(item) for item in value] 121 if name == 'ReportBackground': 122 return self._process_report_background(value) 123 if name == 'TagStatCombine': 124 return [self._process_tag_stat_combine(v) for v in value] 125 if name == 'TagStatLink': 126 return [v for v in [self._process_tag_stat_link(v) for v in value] if v] 127 if name == 'Randomize': 128 return self._process_randomize_value(value) 129 if name == 'MaxErrorLines': 130 return self._process_max_error_lines(value) 131 if name == 'RemoveKeywords': 132 self._validate_remove_keywords(value) 133 if name == 'FlattenKeywords': 134 self._validate_flatten_keywords(value) 135 if name == 'WarnOnSkipped': 136 with LOGGER.cache_only: 137 LOGGER.warn("Option '--warnonskippedfiles' is deprecated and " 138 "has no effect. Nowadays all skipped files are " 139 "reported.") 140 return value 141 142 def _escape_as_data(self, value): 143 return value 144 145 def _process_log_level(self, level): 146 level, visible_level = self._split_log_level(level.upper()) 147 self._opts['VisibleLogLevel'] = visible_level 148 return level 149 150 def _split_log_level(self, level): 151 if ':' in level: 152 level, visible_level = level.split(':', 1) 153 else: 154 visible_level = level 155 self._validate_log_level_and_default(level, visible_level) 156 return level, visible_level 157 158 def _validate_log_level_and_default(self, log_level, default): 159 if log_level not in loggerhelper.LEVELS: 160 raise DataError("Invalid log level '%s'" % log_level) 161 if default not in loggerhelper.LEVELS: 162 raise DataError("Invalid log level '%s'" % default) 163 if not loggerhelper.IsLogged(log_level)(default): 164 raise DataError("Default visible log level '%s' is lower than " 165 "log level '%s'" % (default, log_level)) 166 167 def _process_max_error_lines(self, value): 168 if not value or value.upper() == 'NONE': 169 return None 170 value = self._convert_to_integer('maxerrorlines', value) 171 if value < 10: 172 raise DataError("Option '--maxerrorlines' expected an integer " 173 "value greater that 10 but got '%s'." % value) 174 return value 175 176 def _process_randomize_value(self, original): 177 value = original.lower() 178 if ':' in value: 179 value, seed = value.split(':', 1) 180 else: 181 seed = random.randint(0, sys.maxsize) 182 if value in ('test', 'suite'): 183 value += 's' 184 if value not in ('tests', 'suites', 'none', 'all'): 185 self._raise_invalid_option_value('--randomize', original) 186 try: 187 seed = int(seed) 188 except ValueError: 189 self._raise_invalid_option_value('--randomize', original) 190 return value, seed 191 192 def _raise_invalid_option_value(self, option_name, given_value): 193 raise DataError("Option '%s' does not support value '%s'." 194 % (option_name, given_value)) 195 196 def __getitem__(self, name): 197 if name not in self._opts: 198 raise KeyError("Non-existing option '%s'." % name) 199 if name in self._output_opts: 200 return self._get_output_file(name) 201 return self._opts[name] 202 203 def _get_output_file(self, option): 204 """Returns path of the requested output file and creates needed dirs. 205 206 `option` can be 'Output', 'Log', 'Report', 'XUnit' or 'DebugFile'. 207 """ 208 name = self._opts[option] 209 if not name: 210 return None 211 if option == 'Log' and self._output_disabled(): 212 self['Log'] = None 213 LOGGER.error('Log file is not created if output.xml is disabled.') 214 return None 215 name = self._process_output_name(option, name) 216 path = abspath(os.path.join(self['OutputDir'], name)) 217 self._create_output_dir(os.path.dirname(path), option) 218 return path 219 220 def _process_output_name(self, option, name): 221 base, ext = os.path.splitext(name) 222 if self['TimestampOutputs']: 223 base = '%s-%s' % (base, self.start_timestamp) 224 ext = self._get_output_extension(ext, option) 225 return base + ext 226 227 def _get_output_extension(self, ext, type_): 228 if ext != '': 229 return ext 230 if type_ in ['Output', 'XUnit']: 231 return '.xml' 232 if type_ in ['Log', 'Report']: 233 return '.html' 234 if type_ == 'DebugFile': 235 return '.txt' 236 raise FrameworkError("Invalid output file type: %s" % type_) 237 238 def _create_output_dir(self, path, type_): 239 try: 240 if not os.path.exists(path): 241 os.makedirs(path) 242 except EnvironmentError as err: 243 raise DataError("Creating %s file directory '%s' failed: %s" 244 % (type_.lower(), path, err.strerror)) 245 246 def _process_metadata_or_tagdoc(self, value): 247 if ':' in value: 248 return value.split(':', 1) 249 return value, '' 250 251 def _process_report_background(self, colors): 252 if colors.count(':') not in [1, 2]: 253 raise DataError("Invalid report background colors '%s'." % colors) 254 colors = colors.split(':') 255 if len(colors) == 2: 256 return colors[0], colors[0], colors[1] 257 return tuple(colors) 258 259 def _process_tag_stat_combine(self, pattern): 260 if ':' in pattern: 261 pattern, title = pattern.rsplit(':', 1) 262 else: 263 title = '' 264 return self._format_tag_patterns(pattern), title 265 266 def _format_tag_patterns(self, pattern): 267 for search, replace in [('&', 'AND'), ('AND', ' AND '), ('OR', ' OR '), 268 ('NOT', ' NOT '), ('_', ' ')]: 269 if search in pattern: 270 pattern = pattern.replace(search, replace) 271 while ' ' in pattern: 272 pattern = pattern.replace(' ', ' ') 273 if pattern.startswith(' NOT'): 274 pattern = pattern[1:] 275 return pattern 276 277 def _process_tag_stat_link(self, value): 278 tokens = value.split(':') 279 if len(tokens) >= 3: 280 return tokens[0], ':'.join(tokens[1:-1]), tokens[-1] 281 raise DataError("Invalid format for option '--tagstatlink'. " 282 "Expected 'tag:link:title' but got '%s'." % value) 283 284 def _convert_to_positive_integer_or_default(self, name, value): 285 value = self._convert_to_integer(name, value) 286 return value if value > 0 else self._get_default_value(name) 287 288 def _convert_to_integer(self, name, value): 289 try: 290 return int(value) 291 except ValueError: 292 raise DataError("Option '--%s' expected integer value but got '%s'." 293 % (name.lower(), value)) 294 295 def _get_default_value(self, name): 296 return self._cli_opts[name][1] 297 298 def _validate_remove_keywords(self, values): 299 for value in values: 300 try: 301 KeywordRemover(value) 302 except DataError as err: 303 raise DataError("Invalid value for option '--removekeywords'. %s" % err) 304 305 def _validate_flatten_keywords(self, values): 306 try: 307 validate_flatten_keyword(values) 308 except DataError as err: 309 raise DataError("Invalid value for option '--flattenkeywords'. %s" % err) 310 311 def __contains__(self, setting): 312 return setting in self._cli_opts 313 314 def __unicode__(self): 315 return '\n'.join('%s: %s' % (name, self._opts[name]) 316 for name in sorted(self._opts)) 317 318 @property 319 def output_directory(self): 320 return self['OutputDir'] 321 322 @property 323 def output(self): 324 return self['Output'] 325 326 @property 327 def log(self): 328 return self['Log'] 329 330 @property 331 def report(self): 332 return self['Report'] 333 334 @property 335 def xunit(self): 336 return self['XUnit'] 337 338 @property 339 def log_level(self): 340 return self['LogLevel'] 341 342 @property 343 def split_log(self): 344 return self['SplitLog'] 345 346 @property 347 def status_rc(self): 348 return self['StatusRC'] 349 350 @property 351 def xunit_skip_noncritical(self): 352 return self['XUnitSkipNonCritical'] 353 354 @property 355 def statistics_config(self): 356 return { 357 'suite_stat_level': self['SuiteStatLevel'], 358 'tag_stat_include': self['TagStatInclude'], 359 'tag_stat_exclude': self['TagStatExclude'], 360 'tag_stat_combine': self['TagStatCombine'], 361 'tag_stat_link': self['TagStatLink'], 362 'tag_doc': self['TagDoc'], 363 } 364 365 @property 366 def critical_tags(self): 367 return self['Critical'] 368 369 @property 370 def non_critical_tags(self): 371 return self['NonCritical'] 372 373 @property 374 def remove_keywords(self): 375 return self['RemoveKeywords'] 376 377 @property 378 def flatten_keywords(self): 379 return self['FlattenKeywords'] 380 381 @property 382 def pre_rebot_modifiers(self): 383 return self['PreRebotModifiers'] 384 385 @property 386 def console_colors(self): 387 return self['ConsoleColors'] 388 389 @property 390 def rpa(self): 391 return self['RPA'] 392 393 @rpa.setter 394 def rpa(self, value): 395 self['RPA'] = value 396 397 398class RobotSettings(_BaseSettings): 399 _extra_cli_opts = {'Extension' : ('extension', None), 400 'Output' : ('output', 'output.xml'), 401 'LogLevel' : ('loglevel', 'INFO'), 402 'MaxErrorLines' : ('maxerrorlines', 40), 403 'DryRun' : ('dryrun', False), 404 'ExitOnFailure' : ('exitonfailure', False), 405 'ExitOnError' : ('exitonerror', False), 406 'SkipTeardownOnExit' : ('skipteardownonexit', False), 407 'Randomize' : ('randomize', 'NONE'), 408 'RunEmptySuite' : ('runemptysuite', False), 409 'WarnOnSkipped' : ('warnonskippedfiles', None), 410 'Variables' : ('variable', []), 411 'VariableFiles' : ('variablefile', []), 412 'PreRunModifiers' : ('prerunmodifier', []), 413 'Listeners' : ('listener', []), 414 'ConsoleType' : ('console', 'verbose'), 415 'ConsoleTypeDotted' : ('dotted', False), 416 'ConsoleTypeQuiet' : ('quiet', False), 417 'ConsoleWidth' : ('consolewidth', 78), 418 'ConsoleMarkers' : ('consolemarkers', 'AUTO'), 419 'DebugFile' : ('debugfile', None)} 420 421 def get_rebot_settings(self): 422 settings = RebotSettings() 423 settings.start_timestamp = self.start_timestamp 424 settings._opts.update(self._opts) 425 for name in ['Variables', 'VariableFiles', 'Listeners']: 426 del(settings._opts[name]) 427 for name in ['Include', 'Exclude', 'TestNames', 'SuiteNames', 'Metadata']: 428 settings._opts[name] = [] 429 for name in ['Name', 'Doc']: 430 settings._opts[name] = None 431 settings._opts['Output'] = None 432 settings._opts['LogLevel'] = 'TRACE' 433 settings._opts['ProcessEmptySuite'] = self['RunEmptySuite'] 434 return settings 435 436 def _output_disabled(self): 437 return self.output is None 438 439 def _escape_as_data(self, value): 440 return escape(value) 441 442 @property 443 def listeners(self): 444 return self['Listeners'] 445 446 @property 447 def debug_file(self): 448 return self['DebugFile'] 449 450 @property 451 def suite_config(self): 452 return { 453 'name': self['Name'], 454 'doc': self['Doc'], 455 'metadata': dict(self['Metadata']), 456 'set_tags': self['SetTag'], 457 'include_tags': self['Include'], 458 'exclude_tags': self['Exclude'], 459 'include_suites': self['SuiteNames'], 460 'include_tests': self['TestNames'], 461 'empty_suite_ok': self.run_empty_suite, 462 'randomize_suites': self.randomize_suites, 463 'randomize_tests': self.randomize_tests, 464 'randomize_seed': self.randomize_seed, 465 } 466 467 @property 468 def randomize_seed(self): 469 return self['Randomize'][1] 470 471 @property 472 def randomize_suites(self): 473 return self['Randomize'][0] in ('suites', 'all') 474 475 @property 476 def randomize_tests(self): 477 return self['Randomize'][0] in ('tests', 'all') 478 479 @property 480 def dry_run(self): 481 return self['DryRun'] 482 @property 483 def exit_on_failure(self): 484 return self['ExitOnFailure'] 485 486 @property 487 def exit_on_error(self): 488 return self['ExitOnError'] 489 490 @property 491 def skip_teardown_on_exit(self): 492 return self['SkipTeardownOnExit'] 493 494 @property 495 def console_output_config(self): 496 return { 497 'type': self.console_type, 498 'width': self.console_width, 499 'colors': self.console_colors, 500 'markers': self.console_markers, 501 'stdout': self['StdOut'], 502 'stderr': self['StdErr'] 503 } 504 505 @property 506 def console_type(self): 507 if self['ConsoleTypeQuiet']: 508 return 'quiet' 509 if self['ConsoleTypeDotted']: 510 return 'dotted' 511 return self['ConsoleType'] 512 513 @property 514 def console_width(self): 515 return self['ConsoleWidth'] 516 517 @property 518 def console_markers(self): 519 return self['ConsoleMarkers'] 520 521 @property 522 def max_error_lines(self): 523 return self['MaxErrorLines'] 524 525 @property 526 def pre_run_modifiers(self): 527 return self['PreRunModifiers'] 528 529 @property 530 def run_empty_suite(self): 531 return self['RunEmptySuite'] 532 533 @property 534 def variables(self): 535 return self['Variables'] 536 537 @property 538 def variable_files(self): 539 return self['VariableFiles'] 540 541 @property 542 def extension(self): 543 return self['Extension'] 544 545 546class RebotSettings(_BaseSettings): 547 _extra_cli_opts = {'Output' : ('output', None), 548 'LogLevel' : ('loglevel', 'TRACE'), 549 'ProcessEmptySuite' : ('processemptysuite', False), 550 'StartTime' : ('starttime', None), 551 'EndTime' : ('endtime', None), 552 'Merge' : ('merge', False)} 553 554 def _output_disabled(self): 555 return False 556 557 @property 558 def suite_config(self): 559 return { 560 'name': self['Name'], 561 'doc': self['Doc'], 562 'metadata': dict(self['Metadata']), 563 'set_tags': self['SetTag'], 564 'include_tags': self['Include'], 565 'exclude_tags': self['Exclude'], 566 'include_suites': self['SuiteNames'], 567 'include_tests': self['TestNames'], 568 'empty_suite_ok': self.process_empty_suite, 569 'remove_keywords': self.remove_keywords, 570 'log_level': self['LogLevel'], 571 'critical_tags': self.critical_tags, 572 'non_critical_tags': self.non_critical_tags, 573 'start_time': self['StartTime'], 574 'end_time': self['EndTime'] 575 } 576 577 @property 578 def log_config(self): 579 if not self.log: 580 return {} 581 return { 582 'rpa': self.rpa, 583 'title': html_escape(self['LogTitle'] or ''), 584 'reportURL': self._url_from_path(self.log, self.report), 585 'splitLogBase': os.path.basename(os.path.splitext(self.log)[0]), 586 'defaultLevel': self['VisibleLogLevel'] 587 } 588 589 @property 590 def report_config(self): 591 if not self.report: 592 return {} 593 return { 594 'rpa': self.rpa, 595 'title': html_escape(self['ReportTitle'] or ''), 596 'logURL': self._url_from_path(self.report, self.log), 597 'background' : self._resolve_background_colors(), 598 } 599 600 def _url_from_path(self, source, destination): 601 if not destination: 602 return None 603 return get_link_path(destination, os.path.dirname(source)) 604 605 def _resolve_background_colors(self): 606 colors = self['ReportBackground'] 607 return {'pass': colors[0], 'nonCriticalFail': colors[1], 'fail': colors[2]} 608 609 @property 610 def merge(self): 611 return self['Merge'] 612 613 @property 614 def console_output_config(self): 615 return { 616 'colors': self.console_colors, 617 'stdout': self['StdOut'], 618 'stderr': self['StdErr'] 619 } 620 621 @property 622 def process_empty_suite(self): 623 return self['ProcessEmptySuite'] 624