1#!/usr/bin/env python3 2 3from __future__ import print_function 4 5import time 6import sys 7import re 8from contextlib import contextmanager 9from os import system, path, makedirs, getenv 10from subprocess import check_output, STDOUT, CalledProcessError 11from collections import defaultdict 12import shutil 13 14from termcolor import colored 15 16 17STABLE_VERSION = getenv('WEBOOB_BACKPORT_STABLE', '1.3') 18DEVEL_BRANCH = getenv('WEBOOB_BACKPORT_DEVEL', 'master') 19 20 21@contextmanager 22def log(message, success='done'): 23 print('%s... ' % message, end='', flush=True) 24 start = time.time() 25 try: 26 yield 27 except KeyboardInterrupt: 28 print(colored('abort', 'red')) 29 sys.exit(1) 30 except Exception as e: 31 print(colored('fail: %s' % e, 'red')) 32 raise 33 else: 34 print('%s %s' % (colored(success, 'green'), 35 colored('(%.2fs)' % (time.time() - start), 'blue'))) 36 37 38def create_compat_dir(name): 39 if not path.exists(name): 40 makedirs(name) 41 with open(path.join(name, '__init__.py'), 'w'): 42 pass 43 44 45MANUAL_PORTS = [ 46 'weboob.tools.captcha.virtkeyboard', 47] 48 49MANUAL_PORT_DIR = path.join(path.dirname(__file__), 'stable_backport_data') 50 51 52class Error(object): 53 def __init__(self, filename, linenum, message): 54 self.filename = filename 55 self.linenum = linenum 56 self.message = message 57 self.compat_dir = path.join(path.dirname(filename), 'compat') 58 59 def __repr__(self): 60 return '<%s filename=%r linenum=%s message=%r>' % (type(self).__name__, self.filename, self.linenum, self.message) 61 62 def reimport_module(self, module): 63 # not a weboob module, probably a false positive. 64 if not module.startswith('weboob'): 65 return 66 67 dirname = module.replace('.', '/') 68 filename = dirname + '.py' 69 new_module = module.replace('.', '_') 70 target = path.join(self.compat_dir, '%s.py' % new_module) 71 base_module = '.'.join(module.split('.')[:-1]) 72 73 try: 74 r = check_output('git show %s:%s' % (DEVEL_BRANCH, filename), shell=True, stderr=STDOUT).decode('utf-8') 75 except CalledProcessError: 76 # this file does not exist, perhaps a directory. 77 return 78 79 if module in MANUAL_PORTS: 80 shutil.copyfile(path.join(MANUAL_PORT_DIR, path.basename(target)), target) 81 else: 82 # Copy module from devel to a compat/ sub-module 83 with open(target, 'w') as fp: 84 for line in r.split('\n'): 85 # Replace relative imports to absolute ones 86 m = re.match(r'^from (\.\.?)([\w_\.]+) import (.*)', line) 87 if m: 88 if m.group(1) == '..': 89 base_module = '.'.join(base_module.split('.')[:-1]) 90 fp.write('from %s.%s import %s\n' % (base_module, m.group(2), m.group(3))) 91 continue 92 93 # Inherit all classes by previous ones, if they already existed. 94 m = re.match(r'^class (\w+)\(([\w,\s]+)\):(.*)', line) 95 if m and path.exists(filename) and system('grep "^class %s" %s >/dev/null' % (m.group(1), filename)) == 0: 96 symbol = m.group(1) 97 trailing = m.group(3) 98 fp.write('from %s import %s as _%s\n' % (module, symbol, symbol)) 99 fp.write('class %s(_%s):%s\n' % (symbol, symbol, trailing)) 100 continue 101 102 fp.write('%s\n' % line) 103 104 # Particular case, in devel some imports have been added to 105 # weboob/browser/__init__.py 106 system(r'sed -i -e "s/from weboob.browser import/from weboob.browser.browsers import/g" %s' 107 % self.filename) 108 # Replace import to this module by a relative import to the copy in 109 # compat/ 110 system(r'sed -i -e "%ss/from \([A-Za-z0-9_\.]\+\) import \(.*\)/from .compat.%s import \2/g" %s' 111 % (self.linenum, new_module, self.filename)) 112 113 114def remove_block(name, start): 115 lines = [] 116 with open(name, 'r') as fd: 117 it = iter(fd) 118 for n in range(start - 1): 119 lines.append(next(it)) 120 line = next(it) 121 122 level = len(re.match(r'^( *)', line).group(1)) 123 for line in it: 124 if not line.strip(): 125 continue 126 new = len(re.match(r'^( *)', line).group(1)) 127 if new <= level: 128 lines.append(line) 129 break 130 131 lines.extend(it) 132 133 with open(name, 'w') as fd: 134 fd.write(''.join(lines)) 135 136 137class NoNameInModuleError(Error): 138 def fixup(self): 139 m = re.match(r"No name '(\w+)' in module '([\w\.]+)'", self.message) 140 module = m.group(2) 141 self.reimport_module(module) 142 143 144class ImportErrorError(Error): 145 def fixup(self): 146 m = re.match(r"Unable to import '([\w\.]+)'", self.message) 147 module = m.group(1) 148 self.reimport_module(module) 149 150 151class ManualBackport(Error): 152 def fixup(self): 153 self.reimport_module(self.message) 154 155 156def replace_all(expr, dest): 157 system(r"""for file in $(git ls-files modules | grep '\.py$'); 158 do 159 sed -i -e "s/""" + expr + '/' + dest + """/g" $file 160 done""") 161 162 163def output_lines(cmd): 164 return check_output(cmd, shell=True, stderr=STDOUT).decode('utf-8').rstrip().split('\n') 165 166 167class StableBackport(object): 168 errors = {'E0611': NoNameInModuleError, 169 'E0401': ImportErrorError, 170 } 171 172 def main(self): 173 with log('Removing previous compat files'): 174 system('git rm -q "modules/*/compat/*.py"') 175 176 with log('Copying last version of modules from devel'): 177 system('git checkout --theirs %s modules' % DEVEL_BRANCH) 178 179 with log('Replacing version number'): 180 replace_all(r"""^\(\s*\)\(VERSION\)\( *\)=\( *\)[\"'][0-9]\+\..\+[\"']\(,\?\)$""", 181 r"""\1\2\3=\4'""" + STABLE_VERSION + r"""'\5""") 182 183 with log('Removing staling data'): 184 system('tools/stale_pyc.py') 185 system('find modules -type d -empty -delete') 186 system('git add -u') 187 188 with log('Lookup modules errors'): 189 r = check_output("pylint modules/* -f parseable -E -d all -e no-name-in-module,import-error; exit 0", shell=True, stderr=STDOUT).decode('utf-8') 190 191 dirnames = defaultdict(list) 192 for line in r.split('\n'): 193 m = re.match(r'([\w\./]+):(\d+): \[(\w+)[^\]]+\] (.*)', line) 194 if not m: 195 continue 196 197 filename = m.group(1) 198 linenum = m.group(2) 199 error = m.group(3) 200 msg = m.group(4) 201 202 dirnames[path.dirname(filename)].append(self.errors[error](filename, linenum, msg)) 203 204 with log('Searching manual backports'): 205 for manual in MANUAL_PORTS: 206 r = check_output("grep -nEr '^from %s import ' modules" % manual, shell=True).strip().decode('utf-8') 207 for line in r.split('\n'): 208 m = re.match(r'([\w\./]+):(\d+):.*', line) 209 filename = m.group(1) 210 linenum = m.group(2) 211 target = dirnames[path.dirname(filename)] 212 for err in target: 213 if err.filename == filename and err.linenum == linenum: 214 # an error was already spot on this line 215 break 216 else: 217 target.append(ManualBackport(filename, linenum, manual)) 218 219 for dirname, errors in sorted(dirnames.items()): 220 with log('Fixing up %s errors in %s' % (colored(str(len(errors)), 'magenta'), 221 colored(dirname, 'yellow'))): 222 compat_dirname = path.join(dirname, 'compat') 223 create_compat_dir(compat_dirname) 224 for error in errors: 225 error.fixup() 226 system('git add %s' % compat_dirname) 227 228 system('git add -u') 229 230 231if __name__ == '__main__': 232 StableBackport().main() 233