1"""
2    sphinx.ext.coverage
3    ~~~~~~~~~~~~~~~~~~~
4
5    Check Python modules and C API for coverage.  Mostly written by Josip
6    Dzolonga for the Google Highly Open Participation contest.
7
8    :copyright: Copyright 2007-2021 by the Sphinx team, see AUTHORS.
9    :license: BSD, see LICENSE for details.
10"""
11
12import glob
13import inspect
14import pickle
15import re
16from importlib import import_module
17from os import path
18from typing import IO, Any, Dict, List, Pattern, Set, Tuple
19
20import sphinx
21from sphinx.application import Sphinx
22from sphinx.builders import Builder
23from sphinx.locale import __
24from sphinx.util import logging
25from sphinx.util.console import red  # type: ignore
26from sphinx.util.inspect import safe_getattr
27
28logger = logging.getLogger(__name__)
29
30
31# utility
32def write_header(f: IO, text: str, char: str = '-') -> None:
33    f.write(text + '\n')
34    f.write(char * len(text) + '\n')
35
36
37def compile_regex_list(name: str, exps: str) -> List[Pattern]:
38    lst = []
39    for exp in exps:
40        try:
41            lst.append(re.compile(exp))
42        except Exception:
43            logger.warning(__('invalid regex %r in %s'), exp, name)
44    return lst
45
46
47class CoverageBuilder(Builder):
48    """
49    Evaluates coverage of code in the documentation.
50    """
51    name = 'coverage'
52    epilog = __('Testing of coverage in the sources finished, look at the '
53                'results in %(outdir)s' + path.sep + 'python.txt.')
54
55    def init(self) -> None:
56        self.c_sourcefiles = []  # type: List[str]
57        for pattern in self.config.coverage_c_path:
58            pattern = path.join(self.srcdir, pattern)
59            self.c_sourcefiles.extend(glob.glob(pattern))
60
61        self.c_regexes = []  # type: List[Tuple[str, Pattern]]
62        for (name, exp) in self.config.coverage_c_regexes.items():
63            try:
64                self.c_regexes.append((name, re.compile(exp)))
65            except Exception:
66                logger.warning(__('invalid regex %r in coverage_c_regexes'), exp)
67
68        self.c_ignorexps = {}  # type: Dict[str, List[Pattern]]
69        for (name, exps) in self.config.coverage_ignore_c_items.items():
70            self.c_ignorexps[name] = compile_regex_list('coverage_ignore_c_items',
71                                                        exps)
72        self.mod_ignorexps = compile_regex_list('coverage_ignore_modules',
73                                                self.config.coverage_ignore_modules)
74        self.cls_ignorexps = compile_regex_list('coverage_ignore_classes',
75                                                self.config.coverage_ignore_classes)
76        self.fun_ignorexps = compile_regex_list('coverage_ignore_functions',
77                                                self.config.coverage_ignore_functions)
78        self.py_ignorexps = compile_regex_list('coverage_ignore_pyobjects',
79                                               self.config.coverage_ignore_pyobjects)
80
81    def get_outdated_docs(self) -> str:
82        return 'coverage overview'
83
84    def write(self, *ignored: Any) -> None:
85        self.py_undoc = {}  # type: Dict[str, Dict[str, Any]]
86        self.build_py_coverage()
87        self.write_py_coverage()
88
89        self.c_undoc = {}  # type: Dict[str, Set[Tuple[str, str]]]
90        self.build_c_coverage()
91        self.write_c_coverage()
92
93    def build_c_coverage(self) -> None:
94        # Fetch all the info from the header files
95        c_objects = self.env.domaindata['c']['objects']
96        for filename in self.c_sourcefiles:
97            undoc = set()  # type: Set[Tuple[str, str]]
98            with open(filename) as f:
99                for line in f:
100                    for key, regex in self.c_regexes:
101                        match = regex.match(line)
102                        if match:
103                            name = match.groups()[0]
104                            if name not in c_objects:
105                                for exp in self.c_ignorexps.get(key, []):
106                                    if exp.match(name):
107                                        break
108                                else:
109                                    undoc.add((key, name))
110                            continue
111            if undoc:
112                self.c_undoc[filename] = undoc
113
114    def write_c_coverage(self) -> None:
115        output_file = path.join(self.outdir, 'c.txt')
116        with open(output_file, 'w') as op:
117            if self.config.coverage_write_headline:
118                write_header(op, 'Undocumented C API elements', '=')
119            op.write('\n')
120
121            for filename, undoc in self.c_undoc.items():
122                write_header(op, filename)
123                for typ, name in sorted(undoc):
124                    op.write(' * %-50s [%9s]\n' % (name, typ))
125                    if self.config.coverage_show_missing_items:
126                        if self.app.quiet or self.app.warningiserror:
127                            logger.warning(__('undocumented c api: %s [%s] in file %s'),
128                                           name, typ, filename)
129                        else:
130                            logger.info(red('undocumented  ') + 'c   ' + 'api       ' +
131                                        '%-30s' % (name + " [%9s]" % typ) +
132                                        red(' - in file ') + filename)
133                op.write('\n')
134
135    def ignore_pyobj(self, full_name: str) -> bool:
136        for exp in self.py_ignorexps:
137            if exp.search(full_name):
138                return True
139        return False
140
141    def build_py_coverage(self) -> None:
142        objects = self.env.domaindata['py']['objects']
143        modules = self.env.domaindata['py']['modules']
144
145        skip_undoc = self.config.coverage_skip_undoc_in_source
146
147        for mod_name in modules:
148            ignore = False
149            for exp in self.mod_ignorexps:
150                if exp.match(mod_name):
151                    ignore = True
152                    break
153            if ignore or self.ignore_pyobj(mod_name):
154                continue
155
156            try:
157                mod = import_module(mod_name)
158            except ImportError as err:
159                logger.warning(__('module %s could not be imported: %s'), mod_name, err)
160                self.py_undoc[mod_name] = {'error': err}
161                continue
162
163            funcs = []
164            classes = {}  # type: Dict[str, List[str]]
165
166            for name, obj in inspect.getmembers(mod):
167                # diverse module attributes are ignored:
168                if name[0] == '_':
169                    # begins in an underscore
170                    continue
171                if not hasattr(obj, '__module__'):
172                    # cannot be attributed to a module
173                    continue
174                if obj.__module__ != mod_name:
175                    # is not defined in this module
176                    continue
177
178                full_name = '%s.%s' % (mod_name, name)
179                if self.ignore_pyobj(full_name):
180                    continue
181
182                if inspect.isfunction(obj):
183                    if full_name not in objects:
184                        for exp in self.fun_ignorexps:
185                            if exp.match(name):
186                                break
187                        else:
188                            if skip_undoc and not obj.__doc__:
189                                continue
190                            funcs.append(name)
191                elif inspect.isclass(obj):
192                    for exp in self.cls_ignorexps:
193                        if exp.match(name):
194                            break
195                    else:
196                        if full_name not in objects:
197                            if skip_undoc and not obj.__doc__:
198                                continue
199                            # not documented at all
200                            classes[name] = []
201                            continue
202
203                        attrs = []  # type: List[str]
204
205                        for attr_name in dir(obj):
206                            if attr_name not in obj.__dict__:
207                                continue
208                            try:
209                                attr = safe_getattr(obj, attr_name)
210                            except AttributeError:
211                                continue
212                            if not (inspect.ismethod(attr) or
213                                    inspect.isfunction(attr)):
214                                continue
215                            if attr_name[0] == '_':
216                                # starts with an underscore, ignore it
217                                continue
218                            if skip_undoc and not attr.__doc__:
219                                # skip methods without docstring if wished
220                                continue
221                            full_attr_name = '%s.%s' % (full_name, attr_name)
222                            if self.ignore_pyobj(full_attr_name):
223                                continue
224                            if full_attr_name not in objects:
225                                attrs.append(attr_name)
226                        if attrs:
227                            # some attributes are undocumented
228                            classes[name] = attrs
229
230            self.py_undoc[mod_name] = {'funcs': funcs, 'classes': classes}
231
232    def write_py_coverage(self) -> None:
233        output_file = path.join(self.outdir, 'python.txt')
234        failed = []
235        with open(output_file, 'w') as op:
236            if self.config.coverage_write_headline:
237                write_header(op, 'Undocumented Python objects', '=')
238            keys = sorted(self.py_undoc.keys())
239            for name in keys:
240                undoc = self.py_undoc[name]
241                if 'error' in undoc:
242                    failed.append((name, undoc['error']))
243                else:
244                    if not undoc['classes'] and not undoc['funcs']:
245                        continue
246
247                    write_header(op, name)
248                    if undoc['funcs']:
249                        op.write('Functions:\n')
250                        op.writelines(' * %s\n' % x for x in undoc['funcs'])
251                        if self.config.coverage_show_missing_items:
252                            if self.app.quiet or self.app.warningiserror:
253                                for func in undoc['funcs']:
254                                    logger.warning(
255                                        __('undocumented python function: %s :: %s'),
256                                        name, func)
257                            else:
258                                for func in undoc['funcs']:
259                                    logger.info(red('undocumented  ') + 'py  ' + 'function  ' +
260                                                '%-30s' % func + red(' - in module ') + name)
261                        op.write('\n')
262                    if undoc['classes']:
263                        op.write('Classes:\n')
264                        for class_name, methods in sorted(
265                                undoc['classes'].items()):
266                            if not methods:
267                                op.write(' * %s\n' % class_name)
268                                if self.config.coverage_show_missing_items:
269                                    if self.app.quiet or self.app.warningiserror:
270                                        logger.warning(
271                                            __('undocumented python class: %s :: %s'),
272                                            name, class_name)
273                                    else:
274                                        logger.info(red('undocumented  ') + 'py  ' +
275                                                    'class     ' + '%-30s' % class_name +
276                                                    red(' - in module ') + name)
277                            else:
278                                op.write(' * %s -- missing methods:\n\n' % class_name)
279                                op.writelines('   - %s\n' % x for x in methods)
280                                if self.config.coverage_show_missing_items:
281                                    if self.app.quiet or self.app.warningiserror:
282                                        for meth in methods:
283                                            logger.warning(
284                                                __('undocumented python method:' +
285                                                   ' %s :: %s :: %s'),
286                                                name, class_name, meth)
287                                    else:
288                                        for meth in methods:
289                                            logger.info(red('undocumented  ') + 'py  ' +
290                                                        'method    ' + '%-30s' %
291                                                        (class_name + '.' + meth) +
292                                                        red(' - in module ') + name)
293                        op.write('\n')
294
295            if failed:
296                write_header(op, 'Modules that failed to import')
297                op.writelines(' * %s -- %s\n' % x for x in failed)
298
299    def finish(self) -> None:
300        # dump the coverage data to a pickle file too
301        picklepath = path.join(self.outdir, 'undoc.pickle')
302        with open(picklepath, 'wb') as dumpfile:
303            pickle.dump((self.py_undoc, self.c_undoc), dumpfile)
304
305
306def setup(app: Sphinx) -> Dict[str, Any]:
307    app.add_builder(CoverageBuilder)
308    app.add_config_value('coverage_ignore_modules', [], False)
309    app.add_config_value('coverage_ignore_functions', [], False)
310    app.add_config_value('coverage_ignore_classes', [], False)
311    app.add_config_value('coverage_ignore_pyobjects', [], False)
312    app.add_config_value('coverage_c_path', [], False)
313    app.add_config_value('coverage_c_regexes', {}, False)
314    app.add_config_value('coverage_ignore_c_items', {}, False)
315    app.add_config_value('coverage_write_headline', True, False)
316    app.add_config_value('coverage_skip_undoc_in_source', False, False)
317    app.add_config_value('coverage_show_missing_items', False, False)
318    return {'version': sphinx.__display_version__, 'parallel_read_safe': True}
319