109467b48Spatrickimport argparse 273471bf0Spatrickimport enum 309467b48Spatrickimport os 409467b48Spatrickimport shlex 509467b48Spatrickimport sys 609467b48Spatrick 7097a140dSpatrickimport lit.reports 809467b48Spatrickimport lit.util 909467b48Spatrick 10097a140dSpatrick 11*d415bd75Srobert@enum.unique 1273471bf0Spatrickclass TestOrder(enum.Enum): 13*d415bd75Srobert LEXICAL = 'lexical' 14*d415bd75Srobert RANDOM = 'random' 15*d415bd75Srobert SMART = 'smart' 1673471bf0Spatrick 1773471bf0Spatrick 1809467b48Spatrickdef parse_args(): 19*d415bd75Srobert parser = argparse.ArgumentParser(prog='lit', fromfile_prefix_chars='@') 2009467b48Spatrick parser.add_argument('test_paths', 2109467b48Spatrick nargs='+', 2209467b48Spatrick metavar="TEST_PATH", 2309467b48Spatrick help='File or path to include in the test suite') 2409467b48Spatrick 25097a140dSpatrick parser.add_argument('--version', 26097a140dSpatrick action='version', 27097a140dSpatrick version='%(prog)s ' + lit.__version__) 28097a140dSpatrick 2909467b48Spatrick parser.add_argument("-j", "--threads", "--workers", 3009467b48Spatrick dest="workers", 3109467b48Spatrick metavar="N", 3209467b48Spatrick help="Number of workers used for testing", 3309467b48Spatrick type=_positive_int, 3473471bf0Spatrick default=lit.util.usable_core_count()) 3509467b48Spatrick parser.add_argument("--config-prefix", 3609467b48Spatrick dest="configPrefix", 3709467b48Spatrick metavar="NAME", 3809467b48Spatrick help="Prefix for 'lit' config files") 3909467b48Spatrick parser.add_argument("-D", "--param", 4009467b48Spatrick dest="user_params", 4109467b48Spatrick metavar="NAME=VAL", 4209467b48Spatrick help="Add 'NAME' = 'VAL' to the user defined parameters", 4309467b48Spatrick action="append", 4409467b48Spatrick default=[]) 4509467b48Spatrick 4609467b48Spatrick format_group = parser.add_argument_group("Output Format") 4709467b48Spatrick # FIXME: I find these names very confusing, although I like the 4809467b48Spatrick # functionality. 4909467b48Spatrick format_group.add_argument("-q", "--quiet", 5009467b48Spatrick help="Suppress no error output", 5109467b48Spatrick action="store_true") 5209467b48Spatrick format_group.add_argument("-s", "--succinct", 5373471bf0Spatrick help="Reduce amount of output." 5473471bf0Spatrick " Additionally, show a progress bar," 5573471bf0Spatrick " unless --no-progress-bar is specified.", 5609467b48Spatrick action="store_true") 5709467b48Spatrick format_group.add_argument("-v", "--verbose", 5809467b48Spatrick dest="showOutput", 5909467b48Spatrick help="Show test output for failures", 6009467b48Spatrick action="store_true") 6109467b48Spatrick format_group.add_argument("-vv", "--echo-all-commands", 6209467b48Spatrick dest="echoAllCommands", 6309467b48Spatrick action="store_true", 6409467b48Spatrick help="Echo all commands as they are executed to stdout. In case of " 6509467b48Spatrick "failure, last command shown will be the failing one.") 6609467b48Spatrick format_group.add_argument("-a", "--show-all", 6709467b48Spatrick dest="showAllOutput", 6809467b48Spatrick help="Display all commandlines and output", 6909467b48Spatrick action="store_true") 7009467b48Spatrick format_group.add_argument("-o", "--output", 71097a140dSpatrick type=lit.reports.JsonReport, 7209467b48Spatrick help="Write test results to the provided path", 7309467b48Spatrick metavar="PATH") 7409467b48Spatrick format_group.add_argument("--no-progress-bar", 7509467b48Spatrick dest="useProgressBar", 7609467b48Spatrick help="Do not use curses based progress bar", 7709467b48Spatrick action="store_false") 78097a140dSpatrick 79097a140dSpatrick # Note: this does not generate flags for user-defined result codes. 80097a140dSpatrick success_codes = [c for c in lit.Test.ResultCode.all_codes() 81097a140dSpatrick if not c.isFailure] 82097a140dSpatrick for code in success_codes: 83097a140dSpatrick format_group.add_argument( 84097a140dSpatrick "--show-{}".format(code.name.lower()), 85097a140dSpatrick dest="shown_codes", 86097a140dSpatrick help="Show {} tests ({})".format(code.label.lower(), code.name), 87097a140dSpatrick action="append_const", 88097a140dSpatrick const=code, 89097a140dSpatrick default=[]) 9009467b48Spatrick 9109467b48Spatrick execution_group = parser.add_argument_group("Test Execution") 9209467b48Spatrick execution_group.add_argument("--path", 9309467b48Spatrick help="Additional paths to add to testing environment", 9409467b48Spatrick action="append", 95*d415bd75Srobert default=[], 96*d415bd75Srobert type=os.path.abspath) 9709467b48Spatrick execution_group.add_argument("--vg", 9809467b48Spatrick dest="useValgrind", 9909467b48Spatrick help="Run tests under valgrind", 10009467b48Spatrick action="store_true") 10109467b48Spatrick execution_group.add_argument("--vg-leak", 10209467b48Spatrick dest="valgrindLeakCheck", 10309467b48Spatrick help="Check for memory leaks under valgrind", 10409467b48Spatrick action="store_true") 10509467b48Spatrick execution_group.add_argument("--vg-arg", 10609467b48Spatrick dest="valgrindArgs", 10709467b48Spatrick metavar="ARG", 10809467b48Spatrick help="Specify an extra argument for valgrind", 10909467b48Spatrick action="append", 11009467b48Spatrick default=[]) 11109467b48Spatrick execution_group.add_argument("--time-tests", 11209467b48Spatrick help="Track elapsed wall time for each test", 11309467b48Spatrick action="store_true") 11409467b48Spatrick execution_group.add_argument("--no-execute", 11509467b48Spatrick dest="noExecute", 11609467b48Spatrick help="Don't execute any tests (assume PASS)", 11709467b48Spatrick action="store_true") 11809467b48Spatrick execution_group.add_argument("--xunit-xml-output", 119097a140dSpatrick type=lit.reports.XunitReport, 12009467b48Spatrick help="Write XUnit-compatible XML test reports to the specified file") 121*d415bd75Srobert execution_group.add_argument("--resultdb-output", 122*d415bd75Srobert type=lit.reports.ResultDBReport, 123*d415bd75Srobert help="Write LuCI ResuldDB compatible JSON to the specified file") 12473471bf0Spatrick execution_group.add_argument("--time-trace-output", 12573471bf0Spatrick type=lit.reports.TimeTraceReport, 12673471bf0Spatrick help="Write Chrome tracing compatible JSON to the specified file") 12709467b48Spatrick execution_group.add_argument("--timeout", 12809467b48Spatrick dest="maxIndividualTestTime", 12909467b48Spatrick help="Maximum time to spend running a single test (in seconds). " 13009467b48Spatrick "0 means no time limit. [Default: 0]", 13173471bf0Spatrick type=_non_negative_int) 13209467b48Spatrick execution_group.add_argument("--max-failures", 13309467b48Spatrick help="Stop execution after the given number of failures.", 13409467b48Spatrick type=_positive_int) 13509467b48Spatrick execution_group.add_argument("--allow-empty-runs", 13609467b48Spatrick help="Do not fail the run if all tests are filtered out", 13709467b48Spatrick action="store_true") 13873471bf0Spatrick execution_group.add_argument("--ignore-fail", 13973471bf0Spatrick dest="ignoreFail", 14073471bf0Spatrick action="store_true", 14173471bf0Spatrick help="Exit with status zero even if some tests fail") 14273471bf0Spatrick execution_group.add_argument("--no-indirectly-run-check", 14373471bf0Spatrick dest="indirectlyRunCheck", 14473471bf0Spatrick help="Do not error if a test would not be run if the user had " 14573471bf0Spatrick "specified the containing directory instead of naming the " 14673471bf0Spatrick "test directly.", 14773471bf0Spatrick action="store_false") 14809467b48Spatrick 14909467b48Spatrick selection_group = parser.add_argument_group("Test Selection") 15009467b48Spatrick selection_group.add_argument("--max-tests", 15109467b48Spatrick metavar="N", 15209467b48Spatrick help="Maximum number of tests to run", 15309467b48Spatrick type=_positive_int) 15473471bf0Spatrick selection_group.add_argument("--max-time", 15509467b48Spatrick dest="timeout", 15609467b48Spatrick metavar="N", 15709467b48Spatrick help="Maximum time to spend testing (in seconds)", 15809467b48Spatrick type=_positive_int) 159*d415bd75Srobert selection_group.add_argument("--order", 160*d415bd75Srobert choices=[x.value for x in TestOrder], 161*d415bd75Srobert default=TestOrder.SMART, 162*d415bd75Srobert help="Test order to use (default: smart)") 16373471bf0Spatrick selection_group.add_argument("--shuffle", 164*d415bd75Srobert dest="order", 165*d415bd75Srobert help="Run tests in random order (DEPRECATED: use --order=random)", 166*d415bd75Srobert action="store_const", 167*d415bd75Srobert const=TestOrder.RANDOM) 16873471bf0Spatrick selection_group.add_argument("-i", "--incremental", 169*d415bd75Srobert help="Run failed tests first (DEPRECATED: use --order=smart)", 17009467b48Spatrick action="store_true") 17109467b48Spatrick selection_group.add_argument("--filter", 17209467b48Spatrick metavar="REGEX", 17309467b48Spatrick type=_case_insensitive_regex, 17409467b48Spatrick help="Only run tests with paths matching the given regular expression", 175097a140dSpatrick default=os.environ.get("LIT_FILTER", ".*")) 17673471bf0Spatrick selection_group.add_argument("--filter-out", 17773471bf0Spatrick metavar="REGEX", 17873471bf0Spatrick type=_case_insensitive_regex, 17973471bf0Spatrick help="Filter out tests with paths matching the given regular expression", 18073471bf0Spatrick default=os.environ.get("LIT_FILTER_OUT", "^$")) 18173471bf0Spatrick selection_group.add_argument("--xfail", 18273471bf0Spatrick metavar="LIST", 18373471bf0Spatrick type=_semicolon_list, 18473471bf0Spatrick help="XFAIL tests with paths in the semicolon separated list", 18573471bf0Spatrick default=os.environ.get("LIT_XFAIL", "")) 18673471bf0Spatrick selection_group.add_argument("--xfail-not", 18773471bf0Spatrick metavar="LIST", 18873471bf0Spatrick type=_semicolon_list, 18973471bf0Spatrick help="do not XFAIL tests with paths in the semicolon separated list", 19073471bf0Spatrick default=os.environ.get("LIT_XFAIL_NOT", "")) 19173471bf0Spatrick selection_group.add_argument("--num-shards", 19209467b48Spatrick dest="numShards", 19309467b48Spatrick metavar="M", 19409467b48Spatrick help="Split testsuite into M pieces and only run one", 19509467b48Spatrick type=_positive_int, 19609467b48Spatrick default=os.environ.get("LIT_NUM_SHARDS")) 19709467b48Spatrick selection_group.add_argument("--run-shard", 19809467b48Spatrick dest="runShard", 19909467b48Spatrick metavar="N", 20009467b48Spatrick help="Run shard #N of the testsuite", 20109467b48Spatrick type=_positive_int, 20209467b48Spatrick default=os.environ.get("LIT_RUN_SHARD")) 20309467b48Spatrick 20409467b48Spatrick debug_group = parser.add_argument_group("Debug and Experimental Options") 20509467b48Spatrick debug_group.add_argument("--debug", 20609467b48Spatrick help="Enable debugging (for 'lit' development)", 20709467b48Spatrick action="store_true") 20809467b48Spatrick debug_group.add_argument("--show-suites", 209097a140dSpatrick help="Show discovered test suites and exit", 21009467b48Spatrick action="store_true") 21109467b48Spatrick debug_group.add_argument("--show-tests", 212097a140dSpatrick help="Show all discovered tests and exit", 213097a140dSpatrick action="store_true") 214097a140dSpatrick debug_group.add_argument("--show-used-features", 215097a140dSpatrick help="Show all features used in the test suite (in XFAIL, UNSUPPORTED and REQUIRES) and exit", 21609467b48Spatrick action="store_true") 21709467b48Spatrick 21809467b48Spatrick # LIT is special: environment variables override command line arguments. 21909467b48Spatrick env_args = shlex.split(os.environ.get("LIT_OPTS", "")) 22009467b48Spatrick args = sys.argv[1:] + env_args 22109467b48Spatrick opts = parser.parse_args(args) 22209467b48Spatrick 22309467b48Spatrick # Validate command line options 22409467b48Spatrick if opts.echoAllCommands: 22509467b48Spatrick opts.showOutput = True 22609467b48Spatrick 22773471bf0Spatrick if opts.incremental: 22873471bf0Spatrick print('WARNING: --incremental is deprecated. Failing tests now always run first.') 22973471bf0Spatrick 23009467b48Spatrick if opts.numShards or opts.runShard: 23109467b48Spatrick if not opts.numShards or not opts.runShard: 23209467b48Spatrick parser.error("--num-shards and --run-shard must be used together") 23309467b48Spatrick if opts.runShard > opts.numShards: 23409467b48Spatrick parser.error("--run-shard must be between 1 and --num-shards (inclusive)") 23509467b48Spatrick opts.shard = (opts.runShard, opts.numShards) 23609467b48Spatrick else: 23709467b48Spatrick opts.shard = None 23809467b48Spatrick 239*d415bd75Srobert opts.reports = filter(None, [opts.output, opts.xunit_xml_output, opts.resultdb_output, opts.time_trace_output]) 240097a140dSpatrick 24109467b48Spatrick return opts 24209467b48Spatrick 243097a140dSpatrick 24409467b48Spatrickdef _positive_int(arg): 24509467b48Spatrick return _int(arg, 'positive', lambda i: i > 0) 24609467b48Spatrick 247097a140dSpatrick 24809467b48Spatrickdef _non_negative_int(arg): 24909467b48Spatrick return _int(arg, 'non-negative', lambda i: i >= 0) 25009467b48Spatrick 251097a140dSpatrick 25209467b48Spatrickdef _int(arg, kind, pred): 25309467b48Spatrick desc = "requires {} integer, but found '{}'" 25409467b48Spatrick try: 25509467b48Spatrick i = int(arg) 25609467b48Spatrick except ValueError: 25709467b48Spatrick raise _error(desc, kind, arg) 25809467b48Spatrick if not pred(i): 25909467b48Spatrick raise _error(desc, kind, arg) 26009467b48Spatrick return i 26109467b48Spatrick 262097a140dSpatrick 26309467b48Spatrickdef _case_insensitive_regex(arg): 26409467b48Spatrick import re 26509467b48Spatrick try: 26609467b48Spatrick return re.compile(arg, re.IGNORECASE) 26709467b48Spatrick except re.error as reason: 26809467b48Spatrick raise _error("invalid regular expression: '{}', {}", arg, reason) 26909467b48Spatrick 270097a140dSpatrick 27173471bf0Spatrickdef _semicolon_list(arg): 27273471bf0Spatrick return arg.split(';') 27373471bf0Spatrick 27473471bf0Spatrick 27509467b48Spatrickdef _error(desc, *args): 27609467b48Spatrick msg = desc.format(*args) 27709467b48Spatrick return argparse.ArgumentTypeError(msg) 278