1#!/usr/bin/env python2.7 2# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. 3 4from __future__ import absolute_import 5from __future__ import division 6from __future__ import print_function 7from __future__ import unicode_literals 8import argparse 9import commands 10import subprocess 11import sys 12import re 13import os 14import time 15 16 17# 18# Simple logger 19# 20 21class Log: 22 23 def __init__(self, filename): 24 self.filename = filename 25 self.f = open(self.filename, 'w+', 0) 26 27 def caption(self, str): 28 line = "\n##### %s #####\n" % str 29 if self.f: 30 self.f.write("%s \n" % line) 31 else: 32 print(line) 33 34 def error(self, str): 35 data = "\n\n##### ERROR ##### %s" % str 36 if self.f: 37 self.f.write("%s \n" % data) 38 else: 39 print(data) 40 41 def log(self, str): 42 if self.f: 43 self.f.write("%s \n" % str) 44 else: 45 print(str) 46 47# 48# Shell Environment 49# 50 51 52class Env(object): 53 54 def __init__(self, logfile, tests): 55 self.tests = tests 56 self.log = Log(logfile) 57 58 def shell(self, cmd, path=os.getcwd()): 59 if path: 60 os.chdir(path) 61 62 self.log.log("==== shell session ===========================") 63 self.log.log("%s> %s" % (path, cmd)) 64 status = subprocess.call("cd %s; %s" % (path, cmd), shell=True, 65 stdout=self.log.f, stderr=self.log.f) 66 self.log.log("status = %s" % status) 67 self.log.log("============================================== \n\n") 68 return status 69 70 def GetOutput(self, cmd, path=os.getcwd()): 71 if path: 72 os.chdir(path) 73 74 self.log.log("==== shell session ===========================") 75 self.log.log("%s> %s" % (path, cmd)) 76 status, out = commands.getstatusoutput(cmd) 77 self.log.log("status = %s" % status) 78 self.log.log("out = %s" % out) 79 self.log.log("============================================== \n\n") 80 return status, out 81 82# 83# Pre-commit checker 84# 85 86 87class PreCommitChecker(Env): 88 89 def __init__(self, args): 90 Env.__init__(self, args.logfile, args.tests) 91 self.ignore_failure = args.ignore_failure 92 93 # 94 # Get commands for a given job from the determinator file 95 # 96 def get_commands(self, test): 97 status, out = self.GetOutput( 98 "RATIO=1 build_tools/rocksdb-lego-determinator %s" % test, ".") 99 return status, out 100 101 # 102 # Run a specific CI job 103 # 104 def run_test(self, test): 105 self.log.caption("Running test %s locally" % test) 106 107 # get commands for the CI job determinator 108 status, cmds = self.get_commands(test) 109 if status != 0: 110 self.log.error("Error getting commands for test %s" % test) 111 return False 112 113 # Parse the JSON to extract the commands to run 114 cmds = re.findall("'shell':'([^\']*)'", cmds) 115 116 if len(cmds) == 0: 117 self.log.log("No commands found") 118 return False 119 120 # Run commands 121 for cmd in cmds: 122 # Replace J=<..> with the local environment variable 123 if "J" in os.environ: 124 cmd = cmd.replace("J=1", "J=%s" % os.environ["J"]) 125 cmd = cmd.replace("make ", "make -j%s " % os.environ["J"]) 126 # Run the command 127 status = self.shell(cmd, ".") 128 if status != 0: 129 self.log.error("Error running command %s for test %s" 130 % (cmd, test)) 131 return False 132 133 return True 134 135 # 136 # Run specified CI jobs 137 # 138 def run_tests(self): 139 if not self.tests: 140 self.log.error("Invalid args. Please provide tests") 141 return False 142 143 self.print_separator() 144 self.print_row("TEST", "RESULT") 145 self.print_separator() 146 147 result = True 148 for test in self.tests: 149 start_time = time.time() 150 self.print_test(test) 151 result = self.run_test(test) 152 elapsed_min = (time.time() - start_time) / 60 153 if not result: 154 self.log.error("Error running test %s" % test) 155 self.print_result("FAIL (%dm)" % elapsed_min) 156 if not self.ignore_failure: 157 return False 158 result = False 159 else: 160 self.print_result("PASS (%dm)" % elapsed_min) 161 162 self.print_separator() 163 return result 164 165 # 166 # Print a line 167 # 168 def print_separator(self): 169 print("".ljust(60, "-")) 170 171 # 172 # Print two colums 173 # 174 def print_row(self, c0, c1): 175 print("%s%s" % (c0.ljust(40), c1.ljust(20))) 176 177 def print_test(self, test): 178 print(test.ljust(40), end="") 179 sys.stdout.flush() 180 181 def print_result(self, result): 182 print(result.ljust(20)) 183 184# 185# Main 186# 187parser = argparse.ArgumentParser(description='RocksDB pre-commit checker.') 188 189# --log <logfile> 190parser.add_argument('--logfile', default='/tmp/precommit-check.log', 191 help='Log file. Default is /tmp/precommit-check.log') 192# --ignore_failure 193parser.add_argument('--ignore_failure', action='store_true', default=False, 194 help='Stop when an error occurs') 195# <test ....> 196parser.add_argument('tests', nargs='+', 197 help='CI test(s) to run. e.g: unit punit asan tsan ubsan') 198 199args = parser.parse_args() 200checker = PreCommitChecker(args) 201 202print("Please follow log %s" % checker.log.filename) 203 204if not checker.run_tests(): 205 print("Error running tests. Please check log file %s" 206 % checker.log.filename) 207 sys.exit(1) 208 209sys.exit(0) 210