1# Copyright 2014-2016 The Meson development team
2
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6
7#     http://www.apache.org/licenses/LICENSE-2.0
8
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15"""This is a helper script for IDE developers. It allows you to
16extract information such as list of targets, files, compiler flags,
17tests and so on. All output is in JSON for simple parsing.
18
19Currently only works for the Ninja backend. Others use generated
20project files and don't need this info."""
21
22import collections
23import json
24from . import build, coredata as cdata
25from . import mesonlib
26from .ast import IntrospectionInterpreter, build_target_functions, AstConditionLevel, AstIDGenerator, AstIndentationGenerator, AstJSONPrinter
27from . import mlog
28from .backend import backends
29from .mparser import BaseNode, FunctionNode, ArrayNode, ArgumentNode, StringNode
30from .interpreter import Interpreter
31from pathlib import PurePath
32import typing as T
33import os
34
35def get_meson_info_file(info_dir: str) -> str:
36    return os.path.join(info_dir, 'meson-info.json')
37
38def get_meson_introspection_version() -> str:
39    return '1.0.0'
40
41def get_meson_introspection_required_version() -> T.List[str]:
42    return ['>=1.0', '<2.0']
43
44class IntroCommand:
45    def __init__(self,
46                 desc: str,
47                 func: T.Optional[T.Callable[[], T.Union[dict, list]]] = None,
48                 no_bd: T.Optional[T.Callable[[IntrospectionInterpreter], T.Union[dict, list]]] = None) -> None:
49        self.desc = desc + '.'
50        self.func = func
51        self.no_bd = no_bd
52
53def get_meson_introspection_types(coredata: T.Optional[cdata.CoreData] = None,
54                                  builddata: T.Optional[build.Build] = None,
55                                  backend: T.Optional[backends.Backend] = None,
56                                  sourcedir: T.Optional[str] = None) -> 'T.Mapping[str, IntroCommand]':
57    if backend and builddata:
58        benchmarkdata = backend.create_test_serialisation(builddata.get_benchmarks())
59        testdata = backend.create_test_serialisation(builddata.get_tests())
60        installdata = backend.create_install_data()
61        interpreter = backend.interpreter
62    else:
63        benchmarkdata = testdata = installdata = None
64
65    # Enforce key order for argparse
66    return collections.OrderedDict([
67        ('ast', IntroCommand('Dump the AST of the meson file', no_bd=dump_ast)),
68        ('benchmarks', IntroCommand('List all benchmarks', func=lambda: list_benchmarks(benchmarkdata))),
69        ('buildoptions', IntroCommand('List all build options', func=lambda: list_buildoptions(coredata), no_bd=list_buildoptions_from_source)),
70        ('buildsystem_files', IntroCommand('List files that make up the build system', func=lambda: list_buildsystem_files(builddata, interpreter))),
71        ('dependencies', IntroCommand('List external dependencies', func=lambda: list_deps(coredata), no_bd=list_deps_from_source)),
72        ('scan_dependencies', IntroCommand('Scan for dependencies used in the meson.build file', no_bd=list_deps_from_source)),
73        ('installed', IntroCommand('List all installed files and directories', func=lambda: list_installed(installdata))),
74        ('projectinfo', IntroCommand('Information about projects', func=lambda: list_projinfo(builddata), no_bd=list_projinfo_from_source)),
75        ('targets', IntroCommand('List top level targets', func=lambda: list_targets(builddata, installdata, backend), no_bd=list_targets_from_source)),
76        ('tests', IntroCommand('List all unit tests', func=lambda: list_tests(testdata))),
77    ])
78
79def add_arguments(parser):
80    intro_types = get_meson_introspection_types()
81    for key, val in intro_types.items():
82        flag = '--' + key.replace('_', '-')
83        parser.add_argument(flag, action='store_true', dest=key, default=False, help=val.desc)
84
85    parser.add_argument('--backend', choices=sorted(cdata.backendlist), dest='backend', default='ninja',
86                        help='The backend to use for the --buildoptions introspection.')
87    parser.add_argument('-a', '--all', action='store_true', dest='all', default=False,
88                        help='Print all available information.')
89    parser.add_argument('-i', '--indent', action='store_true', dest='indent', default=False,
90                        help='Enable pretty printed JSON.')
91    parser.add_argument('-f', '--force-object-output', action='store_true', dest='force_dict', default=False,
92                        help='Always use the new JSON format for multiple entries (even for 0 and 1 introspection commands)')
93    parser.add_argument('builddir', nargs='?', default='.', help='The build directory')
94
95def dump_ast(intr: IntrospectionInterpreter) -> T.Dict[str, T.Any]:
96    printer = AstJSONPrinter()
97    intr.ast.accept(printer)
98    return printer.result
99
100def list_installed(installdata):
101    res = {}
102    if installdata is not None:
103        for t in installdata.targets:
104            res[os.path.join(installdata.build_dir, t.fname)] = \
105                os.path.join(installdata.prefix, t.outdir, os.path.basename(t.fname))
106            for alias in t.aliases.keys():
107                res[os.path.join(installdata.build_dir, alias)] = \
108                    os.path.join(installdata.prefix, t.outdir, os.path.basename(alias))
109        for path, installpath, _ in installdata.data:
110            res[path] = os.path.join(installdata.prefix, installpath)
111        for path, installdir, _ in installdata.headers:
112            res[path] = os.path.join(installdata.prefix, installdir, os.path.basename(path))
113        for path, installpath, _ in installdata.man:
114            res[path] = os.path.join(installdata.prefix, installpath)
115        for path, installpath, _, _ in installdata.install_subdirs:
116            res[path] = os.path.join(installdata.prefix, installpath)
117    return res
118
119def list_targets_from_source(intr: IntrospectionInterpreter) -> T.List[T.Dict[str, T.Union[bool, str, T.List[T.Union[str, T.Dict[str, T.Union[str, T.List[str], bool]]]]]]]:
120    tlist = []  # type: T.List[T.Dict[str, T.Union[bool, str, T.List[T.Union[str, T.Dict[str, T.Union[str, T.List[str], bool]]]]]]]
121    for i in intr.targets:
122        sources = []  # type: T.List[str]
123        for n in i['sources']:
124            args = []  # type: T.List[BaseNode]
125            if isinstance(n, FunctionNode):
126                args = list(n.args.arguments)
127                if n.func_name in build_target_functions:
128                    args.pop(0)
129            elif isinstance(n, ArrayNode):
130                args = n.args.arguments
131            elif isinstance(n, ArgumentNode):
132                args = n.arguments
133            for j in args:
134                if isinstance(j, StringNode):
135                    assert isinstance(j.value, str)
136                    sources += [j.value]
137                elif isinstance(j, str):
138                    sources += [j]
139
140        tlist += [{
141            'name': i['name'],
142            'id': i['id'],
143            'type': i['type'],
144            'defined_in': i['defined_in'],
145            'filename': [os.path.join(i['subdir'], x) for x in i['outputs']],
146            'build_by_default': i['build_by_default'],
147            'target_sources': [{
148                'language': 'unknown',
149                'compiler': [],
150                'parameters': [],
151                'sources': [os.path.normpath(os.path.join(os.path.abspath(intr.source_root), i['subdir'], x)) for x in sources],
152                'generated_sources': []
153            }],
154            'subproject': None, # Subprojects are not supported
155            'installed': i['installed']
156        }]
157
158    return tlist
159
160def list_targets(builddata: build.Build, installdata, backend: backends.Backend) -> T.List[T.Dict[str, T.Union[bool, str, T.List[T.Union[str, T.Dict[str, T.Union[str, T.List[str], bool]]]]]]]:
161    tlist = []  # type: T.List[T.Dict[str, T.Union[bool, str, T.List[T.Union[str, T.Dict[str, T.Union[str, T.List[str], bool]]]]]]]
162    build_dir = builddata.environment.get_build_dir()
163    src_dir = builddata.environment.get_source_dir()
164
165    # Fast lookup table for installation files
166    install_lookuptable = {}
167    for i in installdata.targets:
168        out = [os.path.join(installdata.prefix, i.outdir, os.path.basename(i.fname))]
169        out += [os.path.join(installdata.prefix, i.outdir, os.path.basename(x)) for x in i.aliases]
170        install_lookuptable[os.path.basename(i.fname)] = [str(PurePath(x)) for x in out]
171
172    for (idname, target) in builddata.get_targets().items():
173        if not isinstance(target, build.Target):
174            raise RuntimeError('The target object in `builddata.get_targets()` is not of type `build.Target`. Please file a bug with this error message.')
175
176        t = {
177            'name': target.get_basename(),
178            'id': idname,
179            'type': target.get_typename(),
180            'defined_in': os.path.normpath(os.path.join(src_dir, target.subdir, 'meson.build')),
181            'filename': [os.path.join(build_dir, target.subdir, x) for x in target.get_outputs()],
182            'build_by_default': target.build_by_default,
183            'target_sources': backend.get_introspection_data(idname, target),
184            'subproject': target.subproject or None
185        }
186
187        if installdata and target.should_install():
188            t['installed'] = True
189            t['install_filename'] = [install_lookuptable.get(x, [None]) for x in target.get_outputs()]
190            t['install_filename'] = [x for sublist in t['install_filename'] for x in sublist]  # flatten the list
191        else:
192            t['installed'] = False
193        tlist.append(t)
194    return tlist
195
196def list_buildoptions_from_source(intr: IntrospectionInterpreter) -> T.List[T.Dict[str, T.Union[str, bool, int, T.List[str]]]]:
197    subprojects = [i['name'] for i in intr.project_data['subprojects']]
198    return list_buildoptions(intr.coredata, subprojects)
199
200def list_buildoptions(coredata: cdata.CoreData, subprojects: T.Optional[T.List[str]] = None) -> T.List[T.Dict[str, T.Union[str, bool, int, T.List[str]]]]:
201    optlist = []  # type: T.List[T.Dict[str, T.Union[str, bool, int, T.List[str]]]]
202
203    dir_option_names = ['bindir',
204                        'datadir',
205                        'includedir',
206                        'infodir',
207                        'libdir',
208                        'libexecdir',
209                        'localedir',
210                        'localstatedir',
211                        'mandir',
212                        'prefix',
213                        'sbindir',
214                        'sharedstatedir',
215                        'sysconfdir']
216    test_option_names = ['errorlogs',
217                         'stdsplit']
218    core_option_names = [k for k in coredata.builtins if k not in dir_option_names + test_option_names]
219
220    dir_options = {k: o for k, o in coredata.builtins.items() if k in dir_option_names}
221    test_options = {k: o for k, o in coredata.builtins.items() if k in test_option_names}
222    core_options = {k: o for k, o in coredata.builtins.items() if k in core_option_names}
223
224    if subprojects:
225        # Add per subproject built-in options
226        sub_core_options = {}
227        for sub in subprojects:
228            for k, o in core_options.items():
229                if o.yielding:
230                    continue
231                sub_core_options[sub + ':' + k] = o
232        core_options.update(sub_core_options)
233
234    def add_keys(options: T.Dict[str, cdata.UserOption], section: str, machine: str = 'any') -> None:
235        for key in sorted(options.keys()):
236            opt = options[key]
237            optdict = {'name': key, 'value': opt.value, 'section': section, 'machine': machine}
238            if isinstance(opt, cdata.UserStringOption):
239                typestr = 'string'
240            elif isinstance(opt, cdata.UserBooleanOption):
241                typestr = 'boolean'
242            elif isinstance(opt, cdata.UserComboOption):
243                optdict['choices'] = opt.choices
244                typestr = 'combo'
245            elif isinstance(opt, cdata.UserIntegerOption):
246                typestr = 'integer'
247            elif isinstance(opt, cdata.UserArrayOption):
248                typestr = 'array'
249            else:
250                raise RuntimeError("Unknown option type")
251            optdict['type'] = typestr
252            optdict['description'] = opt.description
253            optlist.append(optdict)
254
255    add_keys(core_options, 'core')
256    add_keys(coredata.builtins_per_machine.host, 'core', machine='host')
257    add_keys(
258        {'build.' + k: o for k, o in coredata.builtins_per_machine.build.items()},
259        'core',
260        machine='build',
261    )
262    add_keys(coredata.backend_options, 'backend')
263    add_keys(coredata.base_options, 'base')
264    add_keys(
265        dict(coredata.flatten_lang_iterator(coredata.compiler_options.host.items())),
266        'compiler',
267        machine='host',
268    )
269    add_keys(
270        {
271            'build.' + k: o for k, o in
272            coredata.flatten_lang_iterator(coredata.compiler_options.build.items())
273        },
274        'compiler',
275        machine='build',
276    )
277    add_keys(dir_options, 'directory')
278    add_keys(coredata.user_options, 'user')
279    add_keys(test_options, 'test')
280    return optlist
281
282def find_buildsystem_files_list(src_dir) -> T.List[str]:
283    # I feel dirty about this. But only slightly.
284    filelist = []  # type: T.List[str]
285    for root, _, files in os.walk(src_dir):
286        for f in files:
287            if f == 'meson.build' or f == 'meson_options.txt':
288                filelist.append(os.path.relpath(os.path.join(root, f), src_dir))
289    return filelist
290
291def list_buildsystem_files(builddata: build.Build, interpreter: Interpreter) -> T.List[str]:
292    src_dir = builddata.environment.get_source_dir()
293    filelist = interpreter.get_build_def_files()
294    filelist = [PurePath(src_dir, x).as_posix() for x in filelist]
295    return filelist
296
297def list_deps_from_source(intr: IntrospectionInterpreter) -> T.List[T.Dict[str, T.Union[str, bool]]]:
298    result = []  # type: T.List[T.Dict[str, T.Union[str, bool]]]
299    for i in intr.dependencies:
300        keys = [
301            'name',
302            'required',
303            'version',
304            'has_fallback',
305            'conditional',
306        ]
307        result += [{k: v for k, v in i.items() if k in keys}]
308    return result
309
310def list_deps(coredata: cdata.CoreData) -> T.List[T.Dict[str, T.Union[str, T.List[str]]]]:
311    result = []  # type: T.List[T.Dict[str, T.Union[str, T.List[str]]]]
312    for d in coredata.deps.host.values():
313        if d.found():
314            result += [{'name': d.name,
315                        'version': d.get_version(),
316                        'compile_args': d.get_compile_args(),
317                        'link_args': d.get_link_args()}]
318    return result
319
320def get_test_list(testdata) -> T.List[T.Dict[str, T.Union[str, int, T.List[str], T.Dict[str, str]]]]:
321    result = []  # type: T.List[T.Dict[str, T.Union[str, int, T.List[str], T.Dict[str, str]]]]
322    for t in testdata:
323        to = {}
324        if isinstance(t.fname, str):
325            fname = [t.fname]
326        else:
327            fname = t.fname
328        to['cmd'] = fname + t.cmd_args
329        if isinstance(t.env, build.EnvironmentVariables):
330            to['env'] = t.env.get_env({})
331        else:
332            to['env'] = t.env
333        to['name'] = t.name
334        to['workdir'] = t.workdir
335        to['timeout'] = t.timeout
336        to['suite'] = t.suite
337        to['is_parallel'] = t.is_parallel
338        to['priority'] = t.priority
339        to['protocol'] = str(t.protocol)
340        result.append(to)
341    return result
342
343def list_tests(testdata) -> T.List[T.Dict[str, T.Union[str, int, T.List[str], T.Dict[str, str]]]]:
344    return get_test_list(testdata)
345
346def list_benchmarks(benchdata) -> T.List[T.Dict[str, T.Union[str, int, T.List[str], T.Dict[str, str]]]]:
347    return get_test_list(benchdata)
348
349def list_projinfo(builddata: build.Build) -> T.Dict[str, T.Union[str, T.List[T.Dict[str, str]]]]:
350    result = {'version': builddata.project_version,
351              'descriptive_name': builddata.project_name,
352              'subproject_dir': builddata.subproject_dir}
353    subprojects = []
354    for k, v in builddata.subprojects.items():
355        c = {'name': k,
356             'version': v,
357             'descriptive_name': builddata.projects.get(k)}
358        subprojects.append(c)
359    result['subprojects'] = subprojects
360    return result
361
362def list_projinfo_from_source(intr: IntrospectionInterpreter) -> T.Dict[str, T.Union[str, T.List[T.Dict[str, str]]]]:
363    sourcedir = intr.source_root
364    files = find_buildsystem_files_list(sourcedir)
365    files = [os.path.normpath(x) for x in files]
366
367    for i in intr.project_data['subprojects']:
368        basedir = os.path.join(intr.subproject_dir, i['name'])
369        i['buildsystem_files'] = [x for x in files if x.startswith(basedir)]
370        files = [x for x in files if not x.startswith(basedir)]
371
372    intr.project_data['buildsystem_files'] = files
373    intr.project_data['subproject_dir'] = intr.subproject_dir
374    return intr.project_data
375
376def print_results(options, results: T.Sequence[T.Tuple[str, T.Union[dict, T.List[T.Any]]]], indent: int) -> int:
377    if not results and not options.force_dict:
378        print('No command specified')
379        return 1
380    elif len(results) == 1 and not options.force_dict:
381        # Make to keep the existing output format for a single option
382        print(json.dumps(results[0][1], indent=indent))
383    else:
384        out = {}
385        for i in results:
386            out[i[0]] = i[1]
387        print(json.dumps(out, indent=indent))
388    return 0
389
390def run(options) -> int:
391    datadir = 'meson-private'
392    infodir = 'meson-info'
393    if options.builddir is not None:
394        datadir = os.path.join(options.builddir, datadir)
395        infodir = os.path.join(options.builddir, infodir)
396    indent = 4 if options.indent else None
397    results = []  # type: T.List[T.Tuple[str, T.Union[dict, T.List[T.Any]]]]
398    sourcedir = '.' if options.builddir == 'meson.build' else options.builddir[:-11]
399    intro_types = get_meson_introspection_types(sourcedir=sourcedir)
400
401    if 'meson.build' in [os.path.basename(options.builddir), options.builddir]:
402        # Make sure that log entries in other parts of meson don't interfere with the JSON output
403        mlog.disable()
404        backend = backends.get_backend_from_name(options.backend)
405        intr = IntrospectionInterpreter(sourcedir, '', backend.name, visitors = [AstIDGenerator(), AstIndentationGenerator(), AstConditionLevel()])
406        intr.analyze()
407        # Re-enable logging just in case
408        mlog.enable()
409        for key, val in intro_types.items():
410            if (not options.all and not getattr(options, key, False)) or not val.no_bd:
411                continue
412            results += [(key, val.no_bd(intr))]
413        return print_results(options, results, indent)
414
415    infofile = get_meson_info_file(infodir)
416    if not os.path.isdir(datadir) or not os.path.isdir(infodir) or not os.path.isfile(infofile):
417        print('Current directory is not a meson build directory.\n'
418              'Please specify a valid build dir or change the working directory to it.\n'
419              'It is also possible that the build directory was generated with an old\n'
420              'meson version. Please regenerate it in this case.')
421        return 1
422
423    with open(infofile, 'r') as fp:
424        raw = json.load(fp)
425        intro_vers = raw.get('introspection', {}).get('version', {}).get('full', '0.0.0')
426
427    vers_to_check = get_meson_introspection_required_version()
428    for i in vers_to_check:
429        if not mesonlib.version_compare(intro_vers, i):
430            print('Introspection version {} is not supported. '
431                  'The required version is: {}'
432                  .format(intro_vers, ' and '.join(vers_to_check)))
433            return 1
434
435    # Extract introspection information from JSON
436    for i in intro_types.keys():
437        if not intro_types[i].func:
438            continue
439        if not options.all and not getattr(options, i, False):
440            continue
441        curr = os.path.join(infodir, 'intro-{}.json'.format(i))
442        if not os.path.isfile(curr):
443            print('Introspection file {} does not exist.'.format(curr))
444            return 1
445        with open(curr, 'r') as fp:
446            results += [(i, json.load(fp))]
447
448    return print_results(options, results, indent)
449
450updated_introspection_files = []  # type: T.List[str]
451
452def write_intro_info(intro_info: T.Sequence[T.Tuple[str, T.Union[dict, T.List[T.Any]]]], info_dir: str) -> None:
453    global updated_introspection_files
454    for i in intro_info:
455        out_file = os.path.join(info_dir, 'intro-{}.json'.format(i[0]))
456        tmp_file = os.path.join(info_dir, 'tmp_dump.json')
457        with open(tmp_file, 'w') as fp:
458            json.dump(i[1], fp)
459            fp.flush() # Not sure if this is needed
460        os.replace(tmp_file, out_file)
461        updated_introspection_files += [i[0]]
462
463def generate_introspection_file(builddata: build.Build, backend: backends.Backend) -> None:
464    coredata = builddata.environment.get_coredata()
465    intro_types = get_meson_introspection_types(coredata=coredata, builddata=builddata, backend=backend)
466    intro_info = []  # type: T.List[T.Tuple[str, T.Union[dict, T.List[T.Any]]]]
467
468    for key, val in intro_types.items():
469        if not val.func:
470            continue
471        intro_info += [(key, val.func())]
472
473    write_intro_info(intro_info, builddata.environment.info_dir)
474
475def update_build_options(coredata: cdata.CoreData, info_dir) -> None:
476    intro_info = [
477        ('buildoptions', list_buildoptions(coredata))
478    ]
479
480    write_intro_info(intro_info, info_dir)
481
482def split_version_string(version: str) -> T.Dict[str, T.Union[str, int]]:
483    vers_list = version.split('.')
484    return {
485        'full': version,
486        'major': int(vers_list[0] if len(vers_list) > 0 else 0),
487        'minor': int(vers_list[1] if len(vers_list) > 1 else 0),
488        'patch': int(vers_list[2] if len(vers_list) > 2 else 0)
489    }
490
491def write_meson_info_file(builddata: build.Build, errors: list, build_files_updated: bool = False) -> None:
492    global updated_introspection_files
493    info_dir = builddata.environment.info_dir
494    info_file = get_meson_info_file(info_dir)
495    intro_types = get_meson_introspection_types()
496    intro_info = {}
497
498    for i in intro_types.keys():
499        if not intro_types[i].func:
500            continue
501        intro_info[i] = {
502            'file': 'intro-{}.json'.format(i),
503            'updated': i in updated_introspection_files
504        }
505
506    info_data = {
507        'meson_version': split_version_string(cdata.version),
508        'directories': {
509            'source': builddata.environment.get_source_dir(),
510            'build': builddata.environment.get_build_dir(),
511            'info': info_dir,
512        },
513        'introspection': {
514            'version': split_version_string(get_meson_introspection_version()),
515            'information': intro_info,
516        },
517        'build_files_updated': build_files_updated,
518    }
519
520    if errors:
521        info_data['error'] = True
522        info_data['error_list'] = [x if isinstance(x, str) else str(x) for x in errors]
523    else:
524        info_data['error'] = False
525
526    # Write the data to disc
527    tmp_file = os.path.join(info_dir, 'tmp_dump.json')
528    with open(tmp_file, 'w') as fp:
529        json.dump(info_data, fp)
530        fp.flush()
531    os.replace(tmp_file, info_file)
532