1#!/usr/bin/env python3
2# SPDX-License-Identifier: GPL-2.0
3
4"""
5tdc.py - Linux tc (Traffic Control) unit test driver
6
7Copyright (C) 2017 Lucas Bates <lucasb@mojatatu.com>
8"""
9
10import re
11import os
12import sys
13import argparse
14import importlib
15import json
16import subprocess
17import time
18import traceback
19import random
20from multiprocessing import Pool
21from collections import OrderedDict
22from string import Template
23
24from tdc_config import *
25from tdc_helper import *
26
27import TdcPlugin
28from TdcResults import *
29
30class PluginDependencyException(Exception):
31    def __init__(self, missing_pg):
32        self.missing_pg = missing_pg
33
34class PluginMgrTestFail(Exception):
35    def __init__(self, stage, output, message):
36        self.stage = stage
37        self.output = output
38        self.message = message
39
40class PluginMgr:
41    def __init__(self, argparser):
42        super().__init__()
43        self.plugins = set()
44        self.plugin_instances = []
45        self.failed_plugins = {}
46        self.argparser = argparser
47
48        plugindir = os.getenv('TDC_PLUGIN_DIR', './plugins')
49        for dirpath, dirnames, filenames in os.walk(plugindir):
50            for fn in filenames:
51                if (fn.endswith('.py') and
52                    not fn == '__init__.py' and
53                    not fn.startswith('#') and
54                    not fn.startswith('.#')):
55                    mn = fn[0:-3]
56                    foo = importlib.import_module('plugins.' + mn)
57                    self.plugins.add(mn)
58                    self.plugin_instances[mn] = foo.SubPlugin()
59
60    def load_plugin(self, pgdir, pgname):
61        pgname = pgname[0:-3]
62        self.plugins.add(pgname)
63
64        foo = importlib.import_module('{}.{}'.format(pgdir, pgname))
65
66        # nsPlugin must always be the first one
67        if pgname == "nsPlugin":
68            self.plugin_instances.insert(0, (pgname, foo.SubPlugin()))
69            self.plugin_instances[0][1].check_args(self.args, None)
70        else:
71            self.plugin_instances.append((pgname, foo.SubPlugin()))
72            self.plugin_instances[-1][1].check_args(self.args, None)
73
74    def get_required_plugins(self, testlist):
75        '''
76        Get all required plugins from the list of test cases and return
77        all unique items.
78        '''
79        reqs = set()
80        for t in testlist:
81            try:
82                if 'requires' in t['plugins']:
83                    if isinstance(t['plugins']['requires'], list):
84                        reqs.update(set(t['plugins']['requires']))
85                    else:
86                        reqs.add(t['plugins']['requires'])
87                    t['plugins'] = t['plugins']['requires']
88                else:
89                    t['plugins'] = []
90            except KeyError:
91                t['plugins'] = []
92                continue
93
94        return reqs
95
96    def load_required_plugins(self, reqs, parser, args, remaining):
97        '''
98        Get all required plugins from the list of test cases and load any plugin
99        that is not already enabled.
100        '''
101        pgd = ['plugin-lib', 'plugin-lib-custom']
102        pnf = []
103
104        for r in reqs:
105            if r not in self.plugins:
106                fname = '{}.py'.format(r)
107                source_path = []
108                for d in pgd:
109                    pgpath = '{}/{}'.format(d, fname)
110                    if os.path.isfile(pgpath):
111                        source_path.append(pgpath)
112                if len(source_path) == 0:
113                    print('ERROR: unable to find required plugin {}'.format(r))
114                    pnf.append(fname)
115                    continue
116                elif len(source_path) > 1:
117                    print('WARNING: multiple copies of plugin {} found, using version found')
118                    print('at {}'.format(source_path[0]))
119                pgdir = source_path[0]
120                pgdir = pgdir.split('/')[0]
121                self.load_plugin(pgdir, fname)
122        if len(pnf) > 0:
123            raise PluginDependencyException(pnf)
124
125        parser = self.call_add_args(parser)
126        (args, remaining) = parser.parse_known_args(args=remaining, namespace=args)
127        return args
128
129    def call_pre_suite(self, testcount, testidlist):
130        for (_, pgn_inst) in self.plugin_instances:
131            pgn_inst.pre_suite(testcount, testidlist)
132
133    def call_post_suite(self, index):
134        for (_, pgn_inst) in reversed(self.plugin_instances):
135            pgn_inst.post_suite(index)
136
137    def call_pre_case(self, caseinfo, *, test_skip=False):
138        for (pgn, pgn_inst) in self.plugin_instances:
139            if pgn not in caseinfo['plugins']:
140                continue
141            try:
142                pgn_inst.pre_case(caseinfo, test_skip)
143            except Exception as ee:
144                print('exception {} in call to pre_case for {} plugin'.
145                      format(ee, pgn_inst.__class__))
146                print('test_ordinal is {}'.format(test_ordinal))
147                print('testid is {}'.format(caseinfo['id']))
148                raise
149
150    def call_post_case(self, caseinfo):
151        for (pgn, pgn_inst) in reversed(self.plugin_instances):
152            if pgn not in caseinfo['plugins']:
153                continue
154            pgn_inst.post_case()
155
156    def call_pre_execute(self, caseinfo):
157        for (pgn, pgn_inst) in self.plugin_instances:
158            if pgn not in caseinfo['plugins']:
159                continue
160            pgn_inst.pre_execute()
161
162    def call_post_execute(self, caseinfo):
163        for (pgn, pgn_inst) in reversed(self.plugin_instances):
164            if pgn not in caseinfo['plugins']:
165                continue
166            pgn_inst.post_execute()
167
168    def call_add_args(self, parser):
169        for (pgn, pgn_inst) in self.plugin_instances:
170            parser = pgn_inst.add_args(parser)
171        return parser
172
173    def call_check_args(self, args, remaining):
174        for (pgn, pgn_inst) in self.plugin_instances:
175            pgn_inst.check_args(args, remaining)
176
177    def call_adjust_command(self, caseinfo, stage, command):
178        for (pgn, pgn_inst) in self.plugin_instances:
179            if pgn not in caseinfo['plugins']:
180                continue
181            command = pgn_inst.adjust_command(stage, command)
182        return command
183
184    def set_args(self, args):
185        self.args = args
186
187    @staticmethod
188    def _make_argparser(args):
189        self.argparser = argparse.ArgumentParser(
190            description='Linux TC unit tests')
191
192def replace_keywords(cmd):
193    """
194    For a given executable command, substitute any known
195    variables contained within NAMES with the correct values
196    """
197    tcmd = Template(cmd)
198    subcmd = tcmd.safe_substitute(NAMES)
199    return subcmd
200
201
202def exec_cmd(caseinfo, args, pm, stage, command):
203    """
204    Perform any required modifications on an executable command, then run
205    it in a subprocess and return the results.
206    """
207    if len(command.strip()) == 0:
208        return None, None
209    if '$' in command:
210        command = replace_keywords(command)
211
212    command = pm.call_adjust_command(caseinfo, stage, command)
213    if args.verbose > 0:
214        print('command "{}"'.format(command))
215
216    proc = subprocess.Popen(command,
217        shell=True,
218        stdout=subprocess.PIPE,
219        stderr=subprocess.PIPE,
220        env=ENVIR)
221
222    try:
223        (rawout, serr) = proc.communicate(timeout=NAMES['TIMEOUT'])
224        if proc.returncode != 0 and len(serr) > 0:
225            foutput = serr.decode("utf-8", errors="ignore")
226        else:
227            foutput = rawout.decode("utf-8", errors="ignore")
228    except subprocess.TimeoutExpired:
229        foutput = "Command \"{}\" timed out\n".format(command)
230        proc.returncode = 255
231
232    proc.stdout.close()
233    proc.stderr.close()
234    return proc, foutput
235
236
237def prepare_env(caseinfo, args, pm, stage, prefix, cmdlist, output = None):
238    """
239    Execute the setup/teardown commands for a test case.
240    Optionally terminate test execution if the command fails.
241    """
242    if args.verbose > 0:
243        print('{}'.format(prefix))
244    for cmdinfo in cmdlist:
245        if isinstance(cmdinfo, list):
246            exit_codes = cmdinfo[1:]
247            cmd = cmdinfo[0]
248        else:
249            exit_codes = [0]
250            cmd = cmdinfo
251
252        if not cmd:
253            continue
254
255        (proc, foutput) = exec_cmd(caseinfo, args, pm, stage, cmd)
256
257        if proc and (proc.returncode not in exit_codes):
258            print('', file=sys.stderr)
259            print("{} *** Could not execute: \"{}\"".format(prefix, cmd),
260                  file=sys.stderr)
261            print("\n{} *** Error message: \"{}\"".format(prefix, foutput),
262                  file=sys.stderr)
263            print("returncode {}; expected {}".format(proc.returncode,
264                                                      exit_codes))
265            print("\n{} *** Aborting test run.".format(prefix), file=sys.stderr)
266            print("\n\n{} *** stdout ***".format(proc.stdout), file=sys.stderr)
267            print("\n\n{} *** stderr ***".format(proc.stderr), file=sys.stderr)
268            raise PluginMgrTestFail(
269                stage, output,
270                '"{}" did not complete successfully'.format(prefix))
271
272def verify_by_json(procout, res, tidx, args, pm):
273    try:
274        outputJSON = json.loads(procout)
275    except json.JSONDecodeError:
276        res.set_result(ResultState.fail)
277        res.set_failmsg('Cannot decode verify command\'s output. Is it JSON?')
278        return res
279
280    matchJSON = json.loads(json.dumps(tidx['matchJSON']))
281
282    if type(outputJSON) != type(matchJSON):
283        failmsg = 'Original output and matchJSON value are not the same type: output: {} != matchJSON: {} '
284        failmsg = failmsg.format(type(outputJSON).__name__, type(matchJSON).__name__)
285        res.set_result(ResultState.fail)
286        res.set_failmsg(failmsg)
287        return res
288
289    if len(matchJSON) > len(outputJSON):
290        failmsg = "Your matchJSON value is an array, and it contains more elements than the command under test\'s output:\ncommand output (length: {}):\n{}\nmatchJSON value (length: {}):\n{}"
291        failmsg = failmsg.format(len(outputJSON), outputJSON, len(matchJSON), matchJSON)
292        res.set_result(ResultState.fail)
293        res.set_failmsg(failmsg)
294        return res
295    res = find_in_json(res, outputJSON, matchJSON, 0)
296
297    return res
298
299def find_in_json(res, outputJSONVal, matchJSONVal, matchJSONKey=None):
300    if res.get_result() == ResultState.fail:
301        return res
302
303    if type(matchJSONVal) == list:
304        res = find_in_json_list(res, outputJSONVal, matchJSONVal, matchJSONKey)
305
306    elif type(matchJSONVal) == dict:
307        res = find_in_json_dict(res, outputJSONVal, matchJSONVal)
308    else:
309        res = find_in_json_other(res, outputJSONVal, matchJSONVal, matchJSONKey)
310
311    if res.get_result() != ResultState.fail:
312        res.set_result(ResultState.success)
313        return res
314
315    return res
316
317def find_in_json_list(res, outputJSONVal, matchJSONVal, matchJSONKey=None):
318    if (type(matchJSONVal) != type(outputJSONVal)):
319        failmsg = 'Original output and matchJSON value are not the same type: output: {} != matchJSON: {}'
320        failmsg = failmsg.format(outputJSONVal, matchJSONVal)
321        res.set_result(ResultState.fail)
322        res.set_failmsg(failmsg)
323        return res
324
325    if len(matchJSONVal) > len(outputJSONVal):
326        failmsg = "Your matchJSON value is an array, and it contains more elements than the command under test\'s output:\ncommand output (length: {}):\n{}\nmatchJSON value (length: {}):\n{}"
327        failmsg = failmsg.format(len(outputJSONVal), outputJSONVal, len(matchJSONVal), matchJSONVal)
328        res.set_result(ResultState.fail)
329        res.set_failmsg(failmsg)
330        return res
331
332    for matchJSONIdx, matchJSONVal in enumerate(matchJSONVal):
333        res = find_in_json(res, outputJSONVal[matchJSONIdx], matchJSONVal,
334                           matchJSONKey)
335    return res
336
337def find_in_json_dict(res, outputJSONVal, matchJSONVal):
338    for matchJSONKey, matchJSONVal in matchJSONVal.items():
339        if type(outputJSONVal) == dict:
340            if matchJSONKey not in outputJSONVal:
341                failmsg = 'Key not found in json output: {}: {}\nMatching against output: {}'
342                failmsg = failmsg.format(matchJSONKey, matchJSONVal, outputJSONVal)
343                res.set_result(ResultState.fail)
344                res.set_failmsg(failmsg)
345                return res
346
347        else:
348            failmsg = 'Original output and matchJSON value are not the same type: output: {} != matchJSON: {}'
349            failmsg = failmsg.format(type(outputJSON).__name__, type(matchJSON).__name__)
350            res.set_result(ResultState.fail)
351            res.set_failmsg(failmsg)
352            return rest
353
354        if type(outputJSONVal) == dict and (type(outputJSONVal[matchJSONKey]) == dict or
355                type(outputJSONVal[matchJSONKey]) == list):
356            if len(matchJSONVal) > 0:
357                res = find_in_json(res, outputJSONVal[matchJSONKey], matchJSONVal, matchJSONKey)
358            # handling corner case where matchJSONVal == [] or matchJSONVal == {}
359            else:
360                res = find_in_json_other(res, outputJSONVal, matchJSONVal, matchJSONKey)
361        else:
362            res = find_in_json(res, outputJSONVal, matchJSONVal, matchJSONKey)
363    return res
364
365def find_in_json_other(res, outputJSONVal, matchJSONVal, matchJSONKey=None):
366    if matchJSONKey in outputJSONVal:
367        if matchJSONVal != outputJSONVal[matchJSONKey]:
368            failmsg = 'Value doesn\'t match: {}: {} != {}\nMatching against output: {}'
369            failmsg = failmsg.format(matchJSONKey, matchJSONVal, outputJSONVal[matchJSONKey], outputJSONVal)
370            res.set_result(ResultState.fail)
371            res.set_failmsg(failmsg)
372            return res
373
374    return res
375
376def run_one_test(pm, args, index, tidx):
377    global NAMES
378    ns = NAMES['NS']
379    dev0 = NAMES['DEV0']
380    dev1 = NAMES['DEV1']
381    dummy = NAMES['DUMMY']
382    result = True
383    tresult = ""
384    tap = ""
385    res = TestResult(tidx['id'], tidx['name'])
386    if args.verbose > 0:
387        print("\t====================\n=====> ", end="")
388    print("Test " + tidx["id"] + ": " + tidx["name"])
389
390    if 'skip' in tidx:
391        if tidx['skip'] == 'yes':
392            res = TestResult(tidx['id'], tidx['name'])
393            res.set_result(ResultState.skip)
394            res.set_errormsg('Test case designated as skipped.')
395            pm.call_pre_case(tidx, test_skip=True)
396            pm.call_post_execute(tidx)
397            return res
398
399    if 'dependsOn' in tidx:
400        if (args.verbose > 0):
401            print('probe command for test skip')
402        (p, procout) = exec_cmd(tidx, args, pm, 'execute', tidx['dependsOn'])
403        if p:
404            if (p.returncode != 0):
405                res = TestResult(tidx['id'], tidx['name'])
406                res.set_result(ResultState.skip)
407                res.set_errormsg('probe command: test skipped.')
408                pm.call_pre_case(tidx, test_skip=True)
409                pm.call_post_execute(tidx)
410                return res
411
412    # populate NAMES with TESTID for this test
413    NAMES['TESTID'] = tidx['id']
414    NAMES['NS'] = '{}-{}'.format(NAMES['NS'], tidx['random'])
415    NAMES['DEV0'] = '{}id{}'.format(NAMES['DEV0'], tidx['id'])
416    NAMES['DEV1'] = '{}id{}'.format(NAMES['DEV1'], tidx['id'])
417    NAMES['DUMMY'] = '{}id{}'.format(NAMES['DUMMY'], tidx['id'])
418
419    pm.call_pre_case(tidx)
420    prepare_env(tidx, args, pm, 'setup', "-----> prepare stage", tidx["setup"])
421
422    if (args.verbose > 0):
423        print('-----> execute stage')
424    pm.call_pre_execute(tidx)
425    (p, procout) = exec_cmd(tidx, args, pm, 'execute', tidx["cmdUnderTest"])
426    if p:
427        exit_code = p.returncode
428    else:
429        exit_code = None
430
431    pm.call_post_execute(tidx)
432
433    if (exit_code is None or exit_code != int(tidx["expExitCode"])):
434        print("exit: {!r}".format(exit_code))
435        print("exit: {}".format(int(tidx["expExitCode"])))
436        #print("exit: {!r} {}".format(exit_code, int(tidx["expExitCode"])))
437        res.set_result(ResultState.fail)
438        res.set_failmsg('Command exited with {}, expected {}\n{}'.format(exit_code, tidx["expExitCode"], procout))
439        print(procout)
440    else:
441        if args.verbose > 0:
442            print('-----> verify stage')
443        (p, procout) = exec_cmd(tidx, args, pm, 'verify', tidx["verifyCmd"])
444        if procout:
445            if 'matchJSON' in tidx:
446                verify_by_json(procout, res, tidx, args, pm)
447            elif 'matchPattern' in tidx:
448                match_pattern = re.compile(
449                    str(tidx["matchPattern"]), re.DOTALL | re.MULTILINE)
450                match_index = re.findall(match_pattern, procout)
451                if len(match_index) != int(tidx["matchCount"]):
452                    res.set_result(ResultState.fail)
453                    res.set_failmsg('Could not match regex pattern. Verify command output:\n{}'.format(procout))
454                else:
455                    res.set_result(ResultState.success)
456            else:
457                res.set_result(ResultState.fail)
458                res.set_failmsg('Must specify a match option: matchJSON or matchPattern\n{}'.format(procout))
459        elif int(tidx["matchCount"]) != 0:
460            res.set_result(ResultState.fail)
461            res.set_failmsg('No output generated by verify command.')
462        else:
463            res.set_result(ResultState.success)
464
465    prepare_env(tidx, args, pm, 'teardown', '-----> teardown stage', tidx['teardown'], procout)
466    pm.call_post_case(tidx)
467
468    index += 1
469
470    # remove TESTID from NAMES
471    del(NAMES['TESTID'])
472
473    # Restore names
474    NAMES['NS'] = ns
475    NAMES['DEV0'] = dev0
476    NAMES['DEV1'] = dev1
477    NAMES['DUMMY'] = dummy
478
479    return res
480
481def prepare_run(pm, args, testlist):
482    tcount = len(testlist)
483    emergency_exit = False
484    emergency_exit_message = ''
485
486    try:
487        pm.call_pre_suite(tcount, testlist)
488    except Exception as ee:
489        ex_type, ex, ex_tb = sys.exc_info()
490        print('Exception {} {} (caught in pre_suite).'.
491              format(ex_type, ex))
492        traceback.print_tb(ex_tb)
493        emergency_exit_message = 'EMERGENCY EXIT, call_pre_suite failed with exception {} {}\n'.format(ex_type, ex)
494        emergency_exit = True
495
496    if emergency_exit:
497        pm.call_post_suite(1)
498        return emergency_exit_message
499
500def purge_run(pm, index):
501    pm.call_post_suite(index)
502
503def test_runner(pm, args, filtered_tests):
504    """
505    Driver function for the unit tests.
506
507    Prints information about the tests being run, executes the setup and
508    teardown commands and the command under test itself. Also determines
509    success/failure based on the information in the test case and generates
510    TAP output accordingly.
511    """
512    testlist = filtered_tests
513    tcount = len(testlist)
514    index = 1
515    tap = ''
516    badtest = None
517    stage = None
518
519    tsr = TestSuiteReport()
520
521    for tidx in testlist:
522        if "flower" in tidx["category"] and args.device == None:
523            errmsg = "Tests using the DEV2 variable must define the name of a "
524            errmsg += "physical NIC with the -d option when running tdc.\n"
525            errmsg += "Test has been skipped."
526            if args.verbose > 1:
527                print(errmsg)
528            res = TestResult(tidx['id'], tidx['name'])
529            res.set_result(ResultState.skip)
530            res.set_errormsg(errmsg)
531            tsr.add_resultdata(res)
532            index += 1
533            continue
534        try:
535            badtest = tidx  # in case it goes bad
536            res = run_one_test(pm, args, index, tidx)
537            tsr.add_resultdata(res)
538        except PluginMgrTestFail as pmtf:
539            ex_type, ex, ex_tb = sys.exc_info()
540            stage = pmtf.stage
541            message = pmtf.message
542            output = pmtf.output
543            res = TestResult(tidx['id'], tidx['name'])
544            res.set_result(ResultState.fail)
545            res.set_errormsg(pmtf.message)
546            res.set_failmsg(pmtf.output)
547            tsr.add_resultdata(res)
548            index += 1
549            print(message)
550            print('Exception {} {} (caught in test_runner, running test {} {} {} stage {})'.
551                  format(ex_type, ex, index, tidx['id'], tidx['name'], stage))
552            print('---------------')
553            print('traceback')
554            traceback.print_tb(ex_tb)
555            print('---------------')
556            if stage == 'teardown':
557                print('accumulated output for this test:')
558                if pmtf.output:
559                    print(pmtf.output)
560            print('---------------')
561            break
562        index += 1
563
564    # if we failed in setup or teardown,
565    # fill in the remaining tests with ok-skipped
566    count = index
567
568    if tcount + 1 != count:
569        for tidx in testlist[count - 1:]:
570            res = TestResult(tidx['id'], tidx['name'])
571            res.set_result(ResultState.skip)
572            msg = 'skipped - previous {} failed {} {}'.format(stage,
573                index, badtest.get('id', '--Unknown--'))
574            res.set_errormsg(msg)
575            tsr.add_resultdata(res)
576            count += 1
577
578    if args.pause:
579        print('Want to pause\nPress enter to continue ...')
580        if input(sys.stdin):
581            print('got something on stdin')
582
583    return (index, tsr)
584
585def mp_bins(alltests):
586    serial = []
587    parallel = []
588
589    for test in alltests:
590        if 'nsPlugin' not in test['plugins']:
591            serial.append(test)
592        else:
593            # We can only create one netdevsim device at a time
594            if 'netdevsim/new_device' in str(test['setup']):
595                serial.append(test)
596            else:
597                parallel.append(test)
598
599    return (serial, parallel)
600
601def __mp_runner(tests):
602    (_, tsr) = test_runner(mp_pm, mp_args, tests)
603    return tsr._testsuite
604
605def test_runner_mp(pm, args, alltests):
606    prepare_run(pm, args, alltests)
607
608    (serial, parallel) = mp_bins(alltests)
609
610    batches = [parallel[n : n + 32] for n in range(0, len(parallel), 32)]
611    batches.insert(0, serial)
612
613    print("Executing {} tests in parallel and {} in serial".format(len(parallel), len(serial)))
614    print("Using {} batches and {} workers".format(len(batches), args.mp))
615
616    # We can't pickle these objects so workaround them
617    global mp_pm
618    mp_pm = pm
619
620    global mp_args
621    mp_args = args
622
623    with Pool(args.mp) as p:
624        pres = p.map(__mp_runner, batches)
625
626    tsr = TestSuiteReport()
627    for trs in pres:
628        for res in trs:
629            tsr.add_resultdata(res)
630
631    # Passing an index is not useful in MP
632    purge_run(pm, None)
633
634    return tsr
635
636def test_runner_serial(pm, args, alltests):
637    prepare_run(pm, args, alltests)
638
639    if args.verbose:
640        print("Executing {} tests in serial".format(len(alltests)))
641
642    (index, tsr) = test_runner(pm, args, alltests)
643
644    purge_run(pm, index)
645
646    return tsr
647
648def has_blank_ids(idlist):
649    """
650    Search the list for empty ID fields and return true/false accordingly.
651    """
652    return not(all(k for k in idlist))
653
654
655def load_from_file(filename):
656    """
657    Open the JSON file containing the test cases and return them
658    as list of ordered dictionary objects.
659    """
660    try:
661        with open(filename) as test_data:
662            testlist = json.load(test_data, object_pairs_hook=OrderedDict)
663    except json.JSONDecodeError as jde:
664        print('IGNORING test case file {}\n\tBECAUSE:  {}'.format(filename, jde))
665        testlist = list()
666    else:
667        idlist = get_id_list(testlist)
668        if (has_blank_ids(idlist)):
669            for k in testlist:
670                k['filename'] = filename
671    return testlist
672
673def identity(string):
674    return string
675
676def args_parse():
677    """
678    Create the argument parser.
679    """
680    parser = argparse.ArgumentParser(description='Linux TC unit tests')
681    parser.register('type', None, identity)
682    return parser
683
684
685def set_args(parser):
686    """
687    Set the command line arguments for tdc.
688    """
689    parser.add_argument(
690        '--outfile', type=str,
691        help='Path to the file in which results should be saved. ' +
692        'Default target is the current directory.')
693    parser.add_argument(
694        '-p', '--path', type=str,
695        help='The full path to the tc executable to use')
696    sg = parser.add_argument_group(
697        'selection', 'select which test cases: ' +
698        'files plus directories; filtered by categories plus testids')
699    ag = parser.add_argument_group(
700        'action', 'select action to perform on selected test cases')
701
702    sg.add_argument(
703        '-D', '--directory', nargs='+', metavar='DIR',
704        help='Collect tests from the specified directory(ies) ' +
705        '(default [tc-tests])')
706    sg.add_argument(
707        '-f', '--file', nargs='+', metavar='FILE',
708        help='Run tests from the specified file(s)')
709    sg.add_argument(
710        '-c', '--category', nargs='*', metavar='CATG', default=['+c'],
711        help='Run tests only from the specified category/ies, ' +
712        'or if no category/ies is/are specified, list known categories.')
713    sg.add_argument(
714        '-e', '--execute', nargs='+', metavar='ID',
715        help='Execute the specified test cases with specified IDs')
716    ag.add_argument(
717        '-l', '--list', action='store_true',
718        help='List all test cases, or those only within the specified category')
719    ag.add_argument(
720        '-s', '--show', action='store_true', dest='showID',
721        help='Display the selected test cases')
722    ag.add_argument(
723        '-i', '--id', action='store_true', dest='gen_id',
724        help='Generate ID numbers for new test cases')
725    parser.add_argument(
726        '-v', '--verbose', action='count', default=0,
727        help='Show the commands that are being run')
728    parser.add_argument(
729        '--format', default='tap', const='tap', nargs='?',
730        choices=['none', 'xunit', 'tap'],
731        help='Specify the format for test results. (Default: TAP)')
732    parser.add_argument('-d', '--device',
733                        help='Execute test cases that use a physical device, ' +
734                        'where DEVICE is its name. (If not defined, tests ' +
735                        'that require a physical device will be skipped)')
736    parser.add_argument(
737        '-P', '--pause', action='store_true',
738        help='Pause execution just before post-suite stage')
739    parser.add_argument(
740        '-J', '--multiprocess', type=int, default=1, dest='mp',
741        help='Run tests in parallel whenever possible')
742    return parser
743
744
745def check_default_settings(args, remaining, pm):
746    """
747    Process any arguments overriding the default settings,
748    and ensure the settings are correct.
749    """
750    # Allow for overriding specific settings
751    global NAMES
752
753    if args.path != None:
754        NAMES['TC'] = args.path
755    if args.device != None:
756        NAMES['DEV2'] = args.device
757    if 'TIMEOUT' not in NAMES:
758        NAMES['TIMEOUT'] = None
759    if not os.path.isfile(NAMES['TC']):
760        print("The specified tc path " + NAMES['TC'] + " does not exist.")
761        exit(1)
762
763    pm.call_check_args(args, remaining)
764
765
766def get_id_list(alltests):
767    """
768    Generate a list of all IDs in the test cases.
769    """
770    return [x["id"] for x in alltests]
771
772def check_case_id(alltests):
773    """
774    Check for duplicate test case IDs.
775    """
776    idl = get_id_list(alltests)
777    return [x for x in idl if idl.count(x) > 1]
778
779
780def does_id_exist(alltests, newid):
781    """
782    Check if a given ID already exists in the list of test cases.
783    """
784    idl = get_id_list(alltests)
785    return (any(newid == x for x in idl))
786
787
788def generate_case_ids(alltests):
789    """
790    If a test case has a blank ID field, generate a random hex ID for it
791    and then write the test cases back to disk.
792    """
793    for c in alltests:
794        if (c["id"] == ""):
795            while True:
796                newid = str('{:04x}'.format(random.randrange(16**4)))
797                if (does_id_exist(alltests, newid)):
798                    continue
799                else:
800                    c['id'] = newid
801                    break
802
803    ufilename = []
804    for c in alltests:
805        if ('filename' in c):
806            ufilename.append(c['filename'])
807    ufilename = get_unique_item(ufilename)
808    for f in ufilename:
809        testlist = []
810        for t in alltests:
811            if 'filename' in t:
812                if t['filename'] == f:
813                    del t['filename']
814                    testlist.append(t)
815        outfile = open(f, "w")
816        json.dump(testlist, outfile, indent=4)
817        outfile.write("\n")
818        outfile.close()
819
820def filter_tests_by_id(args, testlist):
821    '''
822    Remove tests from testlist that are not in the named id list.
823    If id list is empty, return empty list.
824    '''
825    newlist = list()
826    if testlist and args.execute:
827        target_ids = args.execute
828
829        if isinstance(target_ids, list) and (len(target_ids) > 0):
830            newlist = list(filter(lambda x: x['id'] in target_ids, testlist))
831    return newlist
832
833def filter_tests_by_category(args, testlist):
834    '''
835    Remove tests from testlist that are not in a named category.
836    '''
837    answer = list()
838    if args.category and testlist:
839        test_ids = list()
840        for catg in set(args.category):
841            if catg == '+c':
842                continue
843            print('considering category {}'.format(catg))
844            for tc in testlist:
845                if catg in tc['category'] and tc['id'] not in test_ids:
846                    answer.append(tc)
847                    test_ids.append(tc['id'])
848
849    return answer
850
851def set_random(alltests):
852    for tidx in alltests:
853        tidx['random'] = random.getrandbits(32)
854
855def get_test_cases(args):
856    """
857    If a test case file is specified, retrieve tests from that file.
858    Otherwise, glob for all json files in subdirectories and load from
859    each one.
860    Also, if requested, filter by category, and add tests matching
861    certain ids.
862    """
863    import fnmatch
864
865    flist = []
866    testdirs = ['tc-tests']
867
868    if args.file:
869        # at least one file was specified - remove the default directory
870        testdirs = []
871
872        for ff in args.file:
873            if not os.path.isfile(ff):
874                print("IGNORING file " + ff + "\n\tBECAUSE does not exist.")
875            else:
876                flist.append(os.path.abspath(ff))
877
878    if args.directory:
879        testdirs = args.directory
880
881    for testdir in testdirs:
882        for root, dirnames, filenames in os.walk(testdir):
883            for filename in fnmatch.filter(filenames, '*.json'):
884                candidate = os.path.abspath(os.path.join(root, filename))
885                if candidate not in testdirs:
886                    flist.append(candidate)
887
888    alltestcases = list()
889    for casefile in flist:
890        alltestcases = alltestcases + (load_from_file(casefile))
891
892    allcatlist = get_test_categories(alltestcases)
893    allidlist = get_id_list(alltestcases)
894
895    testcases_by_cats = get_categorized_testlist(alltestcases, allcatlist)
896    idtestcases = filter_tests_by_id(args, alltestcases)
897    cattestcases = filter_tests_by_category(args, alltestcases)
898
899    cat_ids = [x['id'] for x in cattestcases]
900    if args.execute:
901        if args.category:
902            alltestcases = cattestcases + [x for x in idtestcases if x['id'] not in cat_ids]
903        else:
904            alltestcases = idtestcases
905    else:
906        if cat_ids:
907            alltestcases = cattestcases
908        else:
909            # just accept the existing value of alltestcases,
910            # which has been filtered by file/directory
911            pass
912
913    return allcatlist, allidlist, testcases_by_cats, alltestcases
914
915
916def set_operation_mode(pm, parser, args, remaining):
917    """
918    Load the test case data and process remaining arguments to determine
919    what the script should do for this run, and call the appropriate
920    function.
921    """
922    ucat, idlist, testcases, alltests = get_test_cases(args)
923
924    if args.gen_id:
925        if (has_blank_ids(idlist)):
926            alltests = generate_case_ids(alltests)
927        else:
928            print("No empty ID fields found in test files.")
929        exit(0)
930
931    duplicate_ids = check_case_id(alltests)
932    if (len(duplicate_ids) > 0):
933        print("The following test case IDs are not unique:")
934        print(str(set(duplicate_ids)))
935        print("Please correct them before continuing.")
936        exit(1)
937
938    if args.showID:
939        for atest in alltests:
940            print_test_case(atest)
941        exit(0)
942
943    if isinstance(args.category, list) and (len(args.category) == 0):
944        print("Available categories:")
945        print_sll(ucat)
946        exit(0)
947
948    if args.list:
949        list_test_cases(alltests)
950        exit(0)
951
952    set_random(alltests)
953
954    exit_code = 0 # KSFT_PASS
955    if len(alltests):
956        req_plugins = pm.get_required_plugins(alltests)
957        try:
958            args = pm.load_required_plugins(req_plugins, parser, args, remaining)
959        except PluginDependencyException as pde:
960            print('The following plugins were not found:')
961            print('{}'.format(pde.missing_pg))
962
963        if args.mp > 1:
964            catresults = test_runner_mp(pm, args, alltests)
965        else:
966            catresults = test_runner_serial(pm, args, alltests)
967
968        if catresults.count_failures() != 0:
969            exit_code = 1 # KSFT_FAIL
970        if args.format == 'none':
971            print('Test results output suppression requested\n')
972        else:
973            print('\nAll test results: \n')
974            if args.format == 'xunit':
975                suffix = 'xml'
976                res = catresults.format_xunit()
977            elif args.format == 'tap':
978                suffix = 'tap'
979                res = catresults.format_tap()
980            print(res)
981            print('\n\n')
982            if not args.outfile:
983                fname = 'test-results.{}'.format(suffix)
984            else:
985                fname = args.outfile
986            with open(fname, 'w') as fh:
987                fh.write(res)
988                fh.close()
989                if os.getenv('SUDO_UID') is not None:
990                    os.chown(fname, uid=int(os.getenv('SUDO_UID')),
991                        gid=int(os.getenv('SUDO_GID')))
992    else:
993        print('No tests found\n')
994        exit_code = 4 # KSFT_SKIP
995    exit(exit_code)
996
997def main():
998    """
999    Start of execution; set up argument parser and get the arguments,
1000    and start operations.
1001    """
1002    import resource
1003
1004    if sys.version_info.major < 3 or sys.version_info.minor < 8:
1005        sys.exit("tdc requires at least python 3.8")
1006
1007    resource.setrlimit(resource.RLIMIT_NOFILE, (1048576, 1048576))
1008
1009    parser = args_parse()
1010    parser = set_args(parser)
1011    pm = PluginMgr(parser)
1012    parser = pm.call_add_args(parser)
1013    (args, remaining) = parser.parse_known_args()
1014    args.NAMES = NAMES
1015    args.mp = min(args.mp, 4)
1016    pm.set_args(args)
1017    check_default_settings(args, remaining, pm)
1018    if args.verbose > 2:
1019        print('args is {}'.format(args))
1020
1021    try:
1022        set_operation_mode(pm, parser, args, remaining)
1023    except KeyboardInterrupt:
1024        # Cleanup on Ctrl-C
1025        pm.call_post_suite(None)
1026
1027if __name__ == "__main__":
1028    main()
1029