1#!/usr/bin/env python3
2
3from __future__ import print_function
4
5import time
6import sys
7import re
8from contextlib import contextmanager
9from os import system, path, makedirs, getenv
10from subprocess import check_output, STDOUT, CalledProcessError
11from collections import defaultdict
12import shutil
13
14from termcolor import colored
15
16
17STABLE_VERSION = getenv('WEBOOB_BACKPORT_STABLE', '1.3')
18DEVEL_BRANCH = getenv('WEBOOB_BACKPORT_DEVEL', 'master')
19
20
21@contextmanager
22def log(message, success='done'):
23    print('%s... ' % message, end='', flush=True)
24    start = time.time()
25    try:
26        yield
27    except KeyboardInterrupt:
28        print(colored('abort', 'red'))
29        sys.exit(1)
30    except Exception as e:
31        print(colored('fail: %s' % e, 'red'))
32        raise
33    else:
34        print('%s %s' % (colored(success, 'green'),
35                         colored('(%.2fs)' % (time.time() - start), 'blue')))
36
37
38def create_compat_dir(name):
39    if not path.exists(name):
40        makedirs(name)
41    with open(path.join(name, '__init__.py'), 'w'):
42        pass
43
44
45MANUAL_PORTS = [
46    'weboob.tools.captcha.virtkeyboard',
47]
48
49MANUAL_PORT_DIR = path.join(path.dirname(__file__), 'stable_backport_data')
50
51
52class Error(object):
53    def __init__(self, filename, linenum, message):
54        self.filename = filename
55        self.linenum = linenum
56        self.message = message
57        self.compat_dir = path.join(path.dirname(filename), 'compat')
58
59    def __repr__(self):
60        return '<%s filename=%r linenum=%s message=%r>' % (type(self).__name__, self.filename, self.linenum, self.message)
61
62    def reimport_module(self, module):
63        # not a weboob module, probably a false positive.
64        if not module.startswith('weboob'):
65            return
66
67        dirname = module.replace('.', '/')
68        filename = dirname + '.py'
69        new_module = module.replace('.', '_')
70        target = path.join(self.compat_dir, '%s.py' % new_module)
71        base_module = '.'.join(module.split('.')[:-1])
72
73        try:
74            r = check_output('git show %s:%s' % (DEVEL_BRANCH, filename), shell=True, stderr=STDOUT).decode('utf-8')
75        except CalledProcessError:
76            # this file does not exist, perhaps a directory.
77            return
78
79        if module in MANUAL_PORTS:
80            shutil.copyfile(path.join(MANUAL_PORT_DIR, path.basename(target)), target)
81        else:
82            # Copy module from devel to a compat/ sub-module
83            with open(target, 'w') as fp:
84                for line in r.split('\n'):
85                    # Replace relative imports to absolute ones
86                    m = re.match(r'^from (\.\.?)([\w_\.]+) import (.*)', line)
87                    if m:
88                        if m.group(1) == '..':
89                            base_module = '.'.join(base_module.split('.')[:-1])
90                        fp.write('from %s.%s import %s\n' % (base_module, m.group(2), m.group(3)))
91                        continue
92
93                    # Inherit all classes by previous ones, if they already existed.
94                    m = re.match(r'^class (\w+)\(([\w,\s]+)\):(.*)', line)
95                    if m and path.exists(filename) and system('grep "^class %s" %s >/dev/null' % (m.group(1), filename)) == 0:
96                        symbol = m.group(1)
97                        trailing = m.group(3)
98                        fp.write('from %s import %s as _%s\n' % (module, symbol, symbol))
99                        fp.write('class %s(_%s):%s\n' % (symbol, symbol, trailing))
100                        continue
101
102                    fp.write('%s\n' % line)
103
104        # Particular case, in devel some imports have been added to
105        # weboob/browser/__init__.py
106        system(r'sed -i -e "s/from weboob.browser import/from weboob.browser.browsers import/g" %s'
107               % self.filename)
108        # Replace import to this module by a relative import to the copy in
109        # compat/
110        system(r'sed -i -e "%ss/from \([A-Za-z0-9_\.]\+\) import \(.*\)/from .compat.%s import \2/g" %s'
111               % (self.linenum, new_module, self.filename))
112
113
114def remove_block(name, start):
115    lines = []
116    with open(name, 'r') as fd:
117        it = iter(fd)
118        for n in range(start - 1):
119            lines.append(next(it))
120        line = next(it)
121
122        level = len(re.match(r'^( *)', line).group(1))
123        for line in it:
124            if not line.strip():
125                continue
126            new = len(re.match(r'^( *)', line).group(1))
127            if new <= level:
128                lines.append(line)
129                break
130
131        lines.extend(it)
132
133    with open(name, 'w') as fd:
134        fd.write(''.join(lines))
135
136
137class NoNameInModuleError(Error):
138    def fixup(self):
139        m = re.match(r"No name '(\w+)' in module '([\w\.]+)'", self.message)
140        module = m.group(2)
141        self.reimport_module(module)
142
143
144class ImportErrorError(Error):
145    def fixup(self):
146        m = re.match(r"Unable to import '([\w\.]+)'", self.message)
147        module = m.group(1)
148        self.reimport_module(module)
149
150
151class ManualBackport(Error):
152    def fixup(self):
153        self.reimport_module(self.message)
154
155
156def replace_all(expr, dest):
157    system(r"""for file in $(git ls-files modules | grep '\.py$');
158               do
159                   sed -i -e "s/""" + expr + '/' + dest + """/g" $file
160               done""")
161
162
163def output_lines(cmd):
164    return check_output(cmd, shell=True, stderr=STDOUT).decode('utf-8').rstrip().split('\n')
165
166
167class StableBackport(object):
168    errors = {'E0611': NoNameInModuleError,
169              'E0401': ImportErrorError,
170             }
171
172    def main(self):
173        with log('Removing previous compat files'):
174            system('git rm -q "modules/*/compat/*.py"')
175
176        with log('Copying last version of modules from devel'):
177            system('git checkout --theirs %s modules' % DEVEL_BRANCH)
178
179        with log('Replacing version number'):
180            replace_all(r"""^\(\s*\)\(VERSION\)\( *\)=\( *\)[\"'][0-9]\+\..\+[\"']\(,\?\)$""",
181                        r"""\1\2\3=\4'""" + STABLE_VERSION + r"""'\5""")
182
183        with log('Removing staling data'):
184            system('tools/stale_pyc.py')
185            system('find modules -type d -empty -delete')
186            system('git add -u')
187
188        with log('Lookup modules errors'):
189            r = check_output("pylint modules/* -f parseable -E -d all -e no-name-in-module,import-error; exit 0", shell=True, stderr=STDOUT).decode('utf-8')
190
191        dirnames = defaultdict(list)
192        for line in r.split('\n'):
193            m = re.match(r'([\w\./]+):(\d+): \[(\w+)[^\]]+\] (.*)', line)
194            if not m:
195                continue
196
197            filename = m.group(1)
198            linenum = m.group(2)
199            error = m.group(3)
200            msg = m.group(4)
201
202            dirnames[path.dirname(filename)].append(self.errors[error](filename, linenum, msg))
203
204        with log('Searching manual backports'):
205            for manual in MANUAL_PORTS:
206                r = check_output("grep -nEr '^from %s import ' modules" % manual, shell=True).strip().decode('utf-8')
207                for line in r.split('\n'):
208                    m = re.match(r'([\w\./]+):(\d+):.*', line)
209                    filename = m.group(1)
210                    linenum = m.group(2)
211                    target = dirnames[path.dirname(filename)]
212                    for err in target:
213                        if err.filename == filename and err.linenum == linenum:
214                            # an error was already spot on this line
215                            break
216                    else:
217                        target.append(ManualBackport(filename, linenum, manual))
218
219        for dirname, errors in sorted(dirnames.items()):
220            with log('Fixing up %s errors in %s' % (colored(str(len(errors)), 'magenta'),
221                                                    colored(dirname, 'yellow'))):
222                compat_dirname = path.join(dirname, 'compat')
223                create_compat_dir(compat_dirname)
224                for error in errors:
225                    error.fixup()
226                system('git add %s' % compat_dirname)
227
228        system('git add -u')
229
230
231if __name__ == '__main__':
232    StableBackport().main()
233