1#!/usr/bin/env python
2
3"""
4Static Analyzer qualification infrastructure.
5
6The goal is to test the analyzer against different projects,
7check for failures, compare results, and measure performance.
8
9Repository Directory will contain sources of the projects as well as the
10information on how to build them and the expected output.
11Repository Directory structure:
12   - ProjectMap file
13   - Historical Performance Data
14   - Project Dir1
15     - ReferenceOutput
16   - Project Dir2
17     - ReferenceOutput
18   ..
19Note that the build tree must be inside the project dir.
20
21To test the build of the analyzer one would:
22   - Copy over a copy of the Repository Directory. (TODO: Prefer to ensure that
23     the build directory does not pollute the repository to min network
24     traffic).
25   - Build all projects, until error. Produce logs to report errors.
26   - Compare results.
27
28The files which should be kept around for failure investigations:
29   RepositoryCopy/Project DirI/ScanBuildResults
30   RepositoryCopy/Project DirI/run_static_analyzer.log
31
32Assumptions (TODO: shouldn't need to assume these.):
33   The script is being run from the Repository Directory.
34   The compiler for scan-build and scan-build are in the PATH.
35   export PATH=/Users/zaks/workspace/c2llvm/build/Release+Asserts/bin:$PATH
36
37For more logging, set the  env variables:
38   zaks:TI zaks$ export CCC_ANALYZER_LOG=1
39   zaks:TI zaks$ export CCC_ANALYZER_VERBOSE=1
40
41The list of checkers tested are hardcoded in the Checkers variable.
42For testing additional checkers, use the SA_ADDITIONAL_CHECKERS environment
43variable. It should contain a comma separated list.
44"""
45import CmpRuns
46import SATestUtils
47
48from subprocess import CalledProcessError, check_call
49import argparse
50import csv
51import glob
52import logging
53import math
54import multiprocessing
55import os
56import plistlib
57import shutil
58import sys
59import threading
60import time
61try:
62    import queue
63except ImportError:
64    import Queue as queue
65
66###############################################################################
67# Helper functions.
68###############################################################################
69
70Local = threading.local()
71Local.stdout = sys.stdout
72Local.stderr = sys.stderr
73logging.basicConfig(
74    level=logging.DEBUG,
75    format='%(asctime)s:%(levelname)s:%(name)s: %(message)s')
76
77class StreamToLogger(object):
78    def __init__(self, logger, log_level=logging.INFO):
79        self.logger = logger
80        self.log_level = log_level
81
82    def write(self, buf):
83        # Rstrip in order not to write an extra newline.
84        self.logger.log(self.log_level, buf.rstrip())
85
86    def flush(self):
87        pass
88
89    def fileno(self):
90        return 0
91
92
93def getProjectMapPath():
94    ProjectMapPath = os.path.join(os.path.abspath(os.curdir),
95                                  ProjectMapFile)
96    if not os.path.exists(ProjectMapPath):
97        Local.stdout.write("Error: Cannot find the Project Map file " +
98                           ProjectMapPath +
99                           "\nRunning script for the wrong directory?\n")
100        sys.exit(1)
101    return ProjectMapPath
102
103
104def getProjectDir(ID):
105    return os.path.join(os.path.abspath(os.curdir), ID)
106
107
108def getSBOutputDirName(IsReferenceBuild):
109    if IsReferenceBuild:
110        return SBOutputDirReferencePrefix + SBOutputDirName
111    else:
112        return SBOutputDirName
113
114###############################################################################
115# Configuration setup.
116###############################################################################
117
118
119# Find Clang for static analysis.
120if 'CC' in os.environ:
121    Clang = os.environ['CC']
122else:
123    Clang = SATestUtils.which("clang", os.environ['PATH'])
124if not Clang:
125    print("Error: cannot find 'clang' in PATH")
126    sys.exit(1)
127
128# Number of jobs.
129MaxJobs = int(math.ceil(multiprocessing.cpu_count() * 0.75))
130
131# Project map stores info about all the "registered" projects.
132ProjectMapFile = "projectMap.csv"
133
134# Names of the project specific scripts.
135# The script that downloads the project.
136DownloadScript = "download_project.sh"
137# The script that needs to be executed before the build can start.
138CleanupScript = "cleanup_run_static_analyzer.sh"
139# This is a file containing commands for scan-build.
140BuildScript = "run_static_analyzer.cmd"
141
142# A comment in a build script which disables wrapping.
143NoPrefixCmd = "#NOPREFIX"
144
145# The log file name.
146LogFolderName = "Logs"
147BuildLogName = "run_static_analyzer.log"
148# Summary file - contains the summary of the failures. Ex: This info can be be
149# displayed when buildbot detects a build failure.
150NumOfFailuresInSummary = 10
151FailuresSummaryFileName = "failures.txt"
152
153# The scan-build result directory.
154SBOutputDirName = "ScanBuildResults"
155SBOutputDirReferencePrefix = "Ref"
156
157# The name of the directory storing the cached project source. If this
158# directory does not exist, the download script will be executed.
159# That script should create the "CachedSource" directory and download the
160# project source into it.
161CachedSourceDirName = "CachedSource"
162
163# The name of the directory containing the source code that will be analyzed.
164# Each time a project is analyzed, a fresh copy of its CachedSource directory
165# will be copied to the PatchedSource directory and then the local patches
166# in PatchfileName will be applied (if PatchfileName exists).
167PatchedSourceDirName = "PatchedSource"
168
169# The name of the patchfile specifying any changes that should be applied
170# to the CachedSource before analyzing.
171PatchfileName = "changes_for_analyzer.patch"
172
173# The list of checkers used during analyzes.
174# Currently, consists of all the non-experimental checkers, plus a few alpha
175# checkers we don't want to regress on.
176Checkers = ",".join([
177    "alpha.unix.SimpleStream",
178    "alpha.security.taint",
179    "cplusplus.NewDeleteLeaks",
180    "core",
181    "cplusplus",
182    "deadcode",
183    "security",
184    "unix",
185    "osx",
186    "nullability"
187])
188
189Verbose = 0
190
191###############################################################################
192# Test harness logic.
193###############################################################################
194
195
196def runCleanupScript(Dir, PBuildLogFile):
197    """
198    Run pre-processing script if any.
199    """
200    Cwd = os.path.join(Dir, PatchedSourceDirName)
201    ScriptPath = os.path.join(Dir, CleanupScript)
202    SATestUtils.runScript(ScriptPath, PBuildLogFile, Cwd,
203                          Stdout=Local.stdout, Stderr=Local.stderr)
204
205
206def runDownloadScript(Dir, PBuildLogFile):
207    """
208    Run the script to download the project, if it exists.
209    """
210    ScriptPath = os.path.join(Dir, DownloadScript)
211    SATestUtils.runScript(ScriptPath, PBuildLogFile, Dir,
212                          Stdout=Local.stdout, Stderr=Local.stderr)
213
214
215def downloadAndPatch(Dir, PBuildLogFile):
216    """
217    Download the project and apply the local patchfile if it exists.
218    """
219    CachedSourceDirPath = os.path.join(Dir, CachedSourceDirName)
220
221    # If the we don't already have the cached source, run the project's
222    # download script to download it.
223    if not os.path.exists(CachedSourceDirPath):
224        runDownloadScript(Dir, PBuildLogFile)
225        if not os.path.exists(CachedSourceDirPath):
226            Local.stderr.write("Error: '%s' not found after download.\n" % (
227                               CachedSourceDirPath))
228            exit(1)
229
230    PatchedSourceDirPath = os.path.join(Dir, PatchedSourceDirName)
231
232    # Remove potentially stale patched source.
233    if os.path.exists(PatchedSourceDirPath):
234        shutil.rmtree(PatchedSourceDirPath)
235
236    # Copy the cached source and apply any patches to the copy.
237    shutil.copytree(CachedSourceDirPath, PatchedSourceDirPath, symlinks=True)
238    applyPatch(Dir, PBuildLogFile)
239
240
241def applyPatch(Dir, PBuildLogFile):
242    PatchfilePath = os.path.join(Dir, PatchfileName)
243    PatchedSourceDirPath = os.path.join(Dir, PatchedSourceDirName)
244    if not os.path.exists(PatchfilePath):
245        Local.stdout.write("  No local patches.\n")
246        return
247
248    Local.stdout.write("  Applying patch.\n")
249    try:
250        check_call("patch -p1 < '%s'" % (PatchfilePath),
251                   cwd=PatchedSourceDirPath,
252                   stderr=PBuildLogFile,
253                   stdout=PBuildLogFile,
254                   shell=True)
255    except:
256        Local.stderr.write("Error: Patch failed. See %s for details.\n" % (
257            PBuildLogFile.name))
258        sys.exit(1)
259
260
261def generateAnalyzerConfig(Args):
262    Out = "serialize-stats=true,stable-report-filename=true"
263    if Args.extra_analyzer_config:
264        Out += "," + Args.extra_analyzer_config
265    return Out
266
267
268def runScanBuild(Args, Dir, SBOutputDir, PBuildLogFile):
269    """
270    Build the project with scan-build by reading in the commands and
271    prefixing them with the scan-build options.
272    """
273    BuildScriptPath = os.path.join(Dir, BuildScript)
274    if not os.path.exists(BuildScriptPath):
275        Local.stderr.write(
276            "Error: build script is not defined: %s\n" % BuildScriptPath)
277        sys.exit(1)
278
279    AllCheckers = Checkers
280    if 'SA_ADDITIONAL_CHECKERS' in os.environ:
281        AllCheckers = AllCheckers + ',' + os.environ['SA_ADDITIONAL_CHECKERS']
282
283    # Run scan-build from within the patched source directory.
284    SBCwd = os.path.join(Dir, PatchedSourceDirName)
285
286    SBOptions = "--use-analyzer '%s' " % Clang
287    SBOptions += "-plist-html -o '%s' " % SBOutputDir
288    SBOptions += "-enable-checker " + AllCheckers + " "
289    SBOptions += "--keep-empty "
290    SBOptions += "-analyzer-config '%s' " % generateAnalyzerConfig(Args)
291
292    # Always use ccc-analyze to ensure that we can locate the failures
293    # directory.
294    SBOptions += "--override-compiler "
295    ExtraEnv = {}
296    try:
297        SBCommandFile = open(BuildScriptPath, "r")
298        SBPrefix = "scan-build " + SBOptions + " "
299        for Command in SBCommandFile:
300            Command = Command.strip()
301            if len(Command) == 0:
302                continue
303
304            # Custom analyzer invocation specified by project.
305            # Communicate required information using environment variables
306            # instead.
307            if Command == NoPrefixCmd:
308                SBPrefix = ""
309                ExtraEnv['OUTPUT'] = SBOutputDir
310                ExtraEnv['CC'] = Clang
311                ExtraEnv['ANALYZER_CONFIG'] = generateAnalyzerConfig(Args)
312                continue
313
314            # If using 'make', auto imply a -jX argument
315            # to speed up analysis.  xcodebuild will
316            # automatically use the maximum number of cores.
317            if (Command.startswith("make ") or Command == "make") and \
318                    "-j" not in Command:
319                Command += " -j%d" % MaxJobs
320            SBCommand = SBPrefix + Command
321
322            if Verbose == 1:
323                Local.stdout.write("  Executing: %s\n" % (SBCommand,))
324            check_call(SBCommand, cwd=SBCwd,
325                       stderr=PBuildLogFile,
326                       stdout=PBuildLogFile,
327                       env=dict(os.environ, **ExtraEnv),
328                       shell=True)
329    except CalledProcessError:
330        Local.stderr.write("Error: scan-build failed. Its output was: \n")
331        PBuildLogFile.seek(0)
332        shutil.copyfileobj(PBuildLogFile, Local.stderr)
333        sys.exit(1)
334
335
336def runAnalyzePreprocessed(Args, Dir, SBOutputDir, Mode):
337    """
338    Run analysis on a set of preprocessed files.
339    """
340    if os.path.exists(os.path.join(Dir, BuildScript)):
341        Local.stderr.write(
342            "Error: The preprocessed files project should not contain %s\n" % (
343                BuildScript))
344        raise Exception()
345
346    CmdPrefix = Clang + " --analyze "
347
348    CmdPrefix += "--analyzer-output plist "
349    CmdPrefix += " -Xclang -analyzer-checker=" + Checkers
350    CmdPrefix += " -fcxx-exceptions -fblocks "
351    CmdPrefix += " -Xclang -analyzer-config -Xclang %s "\
352        % generateAnalyzerConfig(Args)
353
354    if (Mode == 2):
355        CmdPrefix += "-std=c++11 "
356
357    PlistPath = os.path.join(Dir, SBOutputDir, "date")
358    FailPath = os.path.join(PlistPath, "failures")
359    os.makedirs(FailPath)
360
361    for FullFileName in glob.glob(Dir + "/*"):
362        FileName = os.path.basename(FullFileName)
363        Failed = False
364
365        # Only run the analyzes on supported files.
366        if SATestUtils.hasNoExtension(FileName):
367            continue
368        if not SATestUtils.isValidSingleInputFile(FileName):
369            Local.stderr.write(
370                "Error: Invalid single input file %s.\n" % (FullFileName,))
371            raise Exception()
372
373        # Build and call the analyzer command.
374        OutputOption = "-o '%s.plist' " % os.path.join(PlistPath, FileName)
375        Command = CmdPrefix + OutputOption + ("'%s'" % FileName)
376        LogFile = open(os.path.join(FailPath, FileName + ".stderr.txt"), "w+b")
377        try:
378            if Verbose == 1:
379                Local.stdout.write("  Executing: %s\n" % (Command,))
380            check_call(Command, cwd=Dir, stderr=LogFile,
381                       stdout=LogFile,
382                       shell=True)
383        except CalledProcessError as e:
384            Local.stderr.write("Error: Analyzes of %s failed. "
385                               "See %s for details."
386                               "Error code %d.\n" % (
387                                   FullFileName, LogFile.name, e.returncode))
388            Failed = True
389        finally:
390            LogFile.close()
391
392        # If command did not fail, erase the log file.
393        if not Failed:
394            os.remove(LogFile.name)
395
396
397def getBuildLogPath(SBOutputDir):
398    return os.path.join(SBOutputDir, LogFolderName, BuildLogName)
399
400
401def removeLogFile(SBOutputDir):
402    BuildLogPath = getBuildLogPath(SBOutputDir)
403    # Clean up the log file.
404    if (os.path.exists(BuildLogPath)):
405        RmCommand = "rm '%s'" % BuildLogPath
406        if Verbose == 1:
407            Local.stdout.write("  Executing: %s\n" % (RmCommand,))
408        check_call(RmCommand, shell=True)
409
410
411def buildProject(Args, Dir, SBOutputDir, ProjectBuildMode, IsReferenceBuild):
412    TBegin = time.time()
413
414    BuildLogPath = getBuildLogPath(SBOutputDir)
415    Local.stdout.write("Log file: %s\n" % (BuildLogPath,))
416    Local.stdout.write("Output directory: %s\n" % (SBOutputDir, ))
417
418    removeLogFile(SBOutputDir)
419
420    # Clean up scan build results.
421    if (os.path.exists(SBOutputDir)):
422        RmCommand = "rm -r '%s'" % SBOutputDir
423        if Verbose == 1:
424            Local.stdout.write("  Executing: %s\n" % (RmCommand,))
425            check_call(RmCommand, shell=True, stdout=Local.stdout,
426                       stderr=Local.stderr)
427    assert(not os.path.exists(SBOutputDir))
428    os.makedirs(os.path.join(SBOutputDir, LogFolderName))
429
430    # Build and analyze the project.
431    with open(BuildLogPath, "wb+") as PBuildLogFile:
432        if (ProjectBuildMode == 1):
433            downloadAndPatch(Dir, PBuildLogFile)
434            runCleanupScript(Dir, PBuildLogFile)
435            runScanBuild(Args, Dir, SBOutputDir, PBuildLogFile)
436        else:
437            runAnalyzePreprocessed(Args, Dir, SBOutputDir, ProjectBuildMode)
438
439        if IsReferenceBuild:
440            runCleanupScript(Dir, PBuildLogFile)
441            normalizeReferenceResults(Dir, SBOutputDir, ProjectBuildMode)
442
443    Local.stdout.write("Build complete (time: %.2f). "
444                       "See the log for more details: %s\n" % (
445                           (time.time() - TBegin), BuildLogPath))
446
447
448def normalizeReferenceResults(Dir, SBOutputDir, ProjectBuildMode):
449    """
450    Make the absolute paths relative in the reference results.
451    """
452    for (DirPath, Dirnames, Filenames) in os.walk(SBOutputDir):
453        for F in Filenames:
454            if (not F.endswith('plist')):
455                continue
456            Plist = os.path.join(DirPath, F)
457            Data = plistlib.readPlist(Plist)
458            PathPrefix = Dir
459            if (ProjectBuildMode == 1):
460                PathPrefix = os.path.join(Dir, PatchedSourceDirName)
461            Paths = [SourceFile[len(PathPrefix) + 1:]
462                     if SourceFile.startswith(PathPrefix)
463                     else SourceFile for SourceFile in Data['files']]
464            Data['files'] = Paths
465
466            # Remove transient fields which change from run to run.
467            for Diag in Data['diagnostics']:
468                if 'HTMLDiagnostics_files' in Diag:
469                    Diag.pop('HTMLDiagnostics_files')
470            if 'clang_version' in Data:
471                Data.pop('clang_version')
472
473            plistlib.writePlist(Data, Plist)
474
475
476def CleanUpEmptyPlists(SBOutputDir):
477    """
478    A plist file is created for each call to the analyzer(each source file).
479    We are only interested on the once that have bug reports,
480    so delete the rest.
481    """
482    for F in glob.glob(SBOutputDir + "/*/*.plist"):
483        P = os.path.join(SBOutputDir, F)
484
485        Data = plistlib.readPlist(P)
486        # Delete empty reports.
487        if not Data['files']:
488            os.remove(P)
489            continue
490
491
492def CleanUpEmptyFolders(SBOutputDir):
493    """
494    Remove empty folders from results, as git would not store them.
495    """
496    Subfolders = glob.glob(SBOutputDir + "/*")
497    for Folder in Subfolders:
498        if not os.listdir(Folder):
499            os.removedirs(Folder)
500
501
502def checkBuild(SBOutputDir):
503    """
504    Given the scan-build output directory, checks if the build failed
505    (by searching for the failures directories). If there are failures, it
506    creates a summary file in the output directory.
507
508    """
509    # Check if there are failures.
510    Failures = glob.glob(SBOutputDir + "/*/failures/*.stderr.txt")
511    TotalFailed = len(Failures)
512    if TotalFailed == 0:
513        CleanUpEmptyPlists(SBOutputDir)
514        CleanUpEmptyFolders(SBOutputDir)
515        Plists = glob.glob(SBOutputDir + "/*/*.plist")
516        Local.stdout.write(
517            "Number of bug reports (non-empty plist files) produced: %d\n" %
518            len(Plists))
519        return
520
521    Local.stderr.write("Error: analysis failed.\n")
522    Local.stderr.write("Total of %d failures discovered.\n" % TotalFailed)
523    if TotalFailed > NumOfFailuresInSummary:
524        Local.stderr.write(
525            "See the first %d below.\n" % NumOfFailuresInSummary)
526        # TODO: Add a line "See the results folder for more."
527
528    Idx = 0
529    for FailLogPathI in Failures:
530        if Idx >= NumOfFailuresInSummary:
531            break
532        Idx += 1
533        Local.stderr.write("\n-- Error #%d -----------\n" % Idx)
534        with open(FailLogPathI, "r") as FailLogI:
535            shutil.copyfileobj(FailLogI, Local.stdout)
536
537    sys.exit(1)
538
539
540def runCmpResults(Dir, Strictness=0):
541    """
542    Compare the warnings produced by scan-build.
543    Strictness defines the success criteria for the test:
544      0 - success if there are no crashes or analyzer failure.
545      1 - success if there are no difference in the number of reported bugs.
546      2 - success if all the bug reports are identical.
547
548    :return success: Whether tests pass according to the Strictness
549    criteria.
550    """
551    TestsPassed = True
552    TBegin = time.time()
553
554    RefDir = os.path.join(Dir, SBOutputDirReferencePrefix + SBOutputDirName)
555    NewDir = os.path.join(Dir, SBOutputDirName)
556
557    # We have to go one level down the directory tree.
558    RefList = glob.glob(RefDir + "/*")
559    NewList = glob.glob(NewDir + "/*")
560
561    # Log folders are also located in the results dir, so ignore them.
562    RefLogDir = os.path.join(RefDir, LogFolderName)
563    if RefLogDir in RefList:
564        RefList.remove(RefLogDir)
565    NewList.remove(os.path.join(NewDir, LogFolderName))
566
567    if len(RefList) != len(NewList):
568        print("Mismatch in number of results folders: %s vs %s" % (
569            RefList, NewList))
570        sys.exit(1)
571
572    # There might be more then one folder underneath - one per each scan-build
573    # command (Ex: one for configure and one for make).
574    if (len(RefList) > 1):
575        # Assume that the corresponding folders have the same names.
576        RefList.sort()
577        NewList.sort()
578
579    # Iterate and find the differences.
580    NumDiffs = 0
581    for P in zip(RefList, NewList):
582        RefDir = P[0]
583        NewDir = P[1]
584
585        assert(RefDir != NewDir)
586        if Verbose == 1:
587            Local.stdout.write("  Comparing Results: %s %s\n" % (
588                               RefDir, NewDir))
589
590        PatchedSourceDirPath = os.path.join(Dir, PatchedSourceDirName)
591        Opts, Args = CmpRuns.generate_option_parser().parse_args(
592            ["--rootA", "", "--rootB", PatchedSourceDirPath])
593        # Scan the results, delete empty plist files.
594        NumDiffs, ReportsInRef, ReportsInNew = \
595            CmpRuns.dumpScanBuildResultsDiff(RefDir, NewDir, Opts,
596                                             deleteEmpty=False,
597                                             Stdout=Local.stdout)
598        if (NumDiffs > 0):
599            Local.stdout.write("Warning: %s differences in diagnostics.\n"
600                               % NumDiffs)
601        if Strictness >= 2 and NumDiffs > 0:
602            Local.stdout.write("Error: Diffs found in strict mode (2).\n")
603            TestsPassed = False
604        elif Strictness >= 1 and ReportsInRef != ReportsInNew:
605            Local.stdout.write("Error: The number of results are different " +
606                               " strict mode (1).\n")
607            TestsPassed = False
608
609    Local.stdout.write("Diagnostic comparison complete (time: %.2f).\n" % (
610                       time.time() - TBegin))
611    return TestsPassed
612
613
614def cleanupReferenceResults(SBOutputDir):
615    """
616    Delete html, css, and js files from reference results. These can
617    include multiple copies of the benchmark source and so get very large.
618    """
619    Extensions = ["html", "css", "js"]
620    for E in Extensions:
621        for F in glob.glob("%s/*/*.%s" % (SBOutputDir, E)):
622            P = os.path.join(SBOutputDir, F)
623            RmCommand = "rm '%s'" % P
624            check_call(RmCommand, shell=True)
625
626    # Remove the log file. It leaks absolute path names.
627    removeLogFile(SBOutputDir)
628
629
630class TestProjectThread(threading.Thread):
631    def __init__(self, Args, TasksQueue, ResultsDiffer, FailureFlag):
632        """
633        :param ResultsDiffer: Used to signify that results differ from
634        the canonical ones.
635        :param FailureFlag: Used to signify a failure during the run.
636        """
637        self.Args = Args
638        self.TasksQueue = TasksQueue
639        self.ResultsDiffer = ResultsDiffer
640        self.FailureFlag = FailureFlag
641        super(TestProjectThread, self).__init__()
642
643        # Needed to gracefully handle interrupts with Ctrl-C
644        self.daemon = True
645
646    def run(self):
647        while not self.TasksQueue.empty():
648            try:
649                ProjArgs = self.TasksQueue.get()
650                Logger = logging.getLogger(ProjArgs[0])
651                Local.stdout = StreamToLogger(Logger, logging.INFO)
652                Local.stderr = StreamToLogger(Logger, logging.ERROR)
653                if not testProject(Args, *ProjArgs):
654                    self.ResultsDiffer.set()
655                self.TasksQueue.task_done()
656            except:
657                self.FailureFlag.set()
658                raise
659
660
661def testProject(Args, ID, ProjectBuildMode, IsReferenceBuild=False, Strictness=0):
662    """
663    Test a given project.
664    :return TestsPassed: Whether tests have passed according
665    to the :param Strictness: criteria.
666    """
667    Local.stdout.write(" \n\n--- Building project %s\n" % (ID,))
668
669    TBegin = time.time()
670
671    Dir = getProjectDir(ID)
672    if Verbose == 1:
673        Local.stdout.write("  Build directory: %s.\n" % (Dir,))
674
675    # Set the build results directory.
676    RelOutputDir = getSBOutputDirName(IsReferenceBuild)
677    SBOutputDir = os.path.join(Dir, RelOutputDir)
678
679    buildProject(Args, Dir, SBOutputDir, ProjectBuildMode, IsReferenceBuild)
680
681    checkBuild(SBOutputDir)
682
683    if IsReferenceBuild:
684        cleanupReferenceResults(SBOutputDir)
685        TestsPassed = True
686    else:
687        TestsPassed = runCmpResults(Dir, Strictness)
688
689    Local.stdout.write("Completed tests for project %s (time: %.2f).\n" % (
690                       ID, (time.time() - TBegin)))
691    return TestsPassed
692
693
694def projectFileHandler():
695    return open(getProjectMapPath(), "rb")
696
697
698def iterateOverProjects(PMapFile):
699    """
700    Iterate over all projects defined in the project file handler `PMapFile`
701    from the start.
702    """
703    PMapFile.seek(0)
704    for I in csv.reader(PMapFile):
705        if (SATestUtils.isCommentCSVLine(I)):
706            continue
707        yield I
708
709
710def validateProjectFile(PMapFile):
711    """
712    Validate project file.
713    """
714    for I in iterateOverProjects(PMapFile):
715        if len(I) != 2:
716            print("Error: Rows in the ProjectMapFile should have 2 entries.")
717            raise Exception()
718        if I[1] not in ('0', '1', '2'):
719            print("Error: Second entry in the ProjectMapFile should be 0" \
720                  " (single file), 1 (project), or 2(single file c++11).")
721            raise Exception()
722
723def singleThreadedTestAll(Args, ProjectsToTest):
724    """
725    Run all projects.
726    :return: whether tests have passed.
727    """
728    Success = True
729    for ProjArgs in ProjectsToTest:
730        Success &= testProject(Args, *ProjArgs)
731    return Success
732
733def multiThreadedTestAll(Args, ProjectsToTest, Jobs):
734    """
735    Run each project in a separate thread.
736
737    This is OK despite GIL, as testing is blocked
738    on launching external processes.
739
740    :return: whether tests have passed.
741    """
742    TasksQueue = queue.Queue()
743
744    for ProjArgs in ProjectsToTest:
745        TasksQueue.put(ProjArgs)
746
747    ResultsDiffer = threading.Event()
748    FailureFlag = threading.Event()
749
750    for i in range(Jobs):
751        T = TestProjectThread(Args, TasksQueue, ResultsDiffer, FailureFlag)
752        T.start()
753
754    # Required to handle Ctrl-C gracefully.
755    while TasksQueue.unfinished_tasks:
756        time.sleep(0.1)  # Seconds.
757        if FailureFlag.is_set():
758            Local.stderr.write("Test runner crashed\n")
759            sys.exit(1)
760    return not ResultsDiffer.is_set()
761
762
763def testAll(Args):
764    ProjectsToTest = []
765
766    with projectFileHandler() as PMapFile:
767        validateProjectFile(PMapFile)
768
769        # Test the projects.
770        for (ProjName, ProjBuildMode) in iterateOverProjects(PMapFile):
771            ProjectsToTest.append((ProjName,
772                                  int(ProjBuildMode),
773                                  Args.regenerate,
774                                  Args.strictness))
775    if Args.jobs <= 1:
776        return singleThreadedTestAll(Args, ProjectsToTest)
777    else:
778        return multiThreadedTestAll(Args, ProjectsToTest, Args.jobs)
779
780
781if __name__ == '__main__':
782    # Parse command line arguments.
783    Parser = argparse.ArgumentParser(
784        description='Test the Clang Static Analyzer.')
785    Parser.add_argument('--strictness', dest='strictness', type=int, default=0,
786                        help='0 to fail on runtime errors, 1 to fail when the \
787                             number of found bugs are different from the \
788                             reference, 2 to fail on any difference from the \
789                             reference. Default is 0.')
790    Parser.add_argument('-r', dest='regenerate', action='store_true',
791                        default=False, help='Regenerate reference output.')
792    Parser.add_argument('-j', '--jobs', dest='jobs', type=int,
793                        default=0,
794                        help='Number of projects to test concurrently')
795    Parser.add_argument('--extra-analyzer-config', dest='extra_analyzer_config',
796                        type=str,
797                        default="",
798                        help="Arguments passed to to -analyzer-config")
799    Args = Parser.parse_args()
800
801    TestsPassed = testAll(Args)
802    if not TestsPassed:
803        print("ERROR: Tests failed.")
804        sys.exit(42)
805