1#!/usr/bin/env python3
2# SPDX-License-Identifier: GPL-2.0+
3#
4# Author: Masahiro Yamada <yamada.m@jp.panasonic.com>
5#
6
7"""
8Converter from Kconfig and MAINTAINERS to a board database.
9
10Run 'tools/genboardscfg.py' to create a board database.
11
12Run 'tools/genboardscfg.py -h' for available options.
13"""
14
15import errno
16import fnmatch
17import glob
18import multiprocessing
19import optparse
20import os
21import sys
22import tempfile
23import time
24
25from buildman import kconfiglib
26
27### constant variables ###
28OUTPUT_FILE = 'boards.cfg'
29CONFIG_DIR = 'configs'
30SLEEP_TIME = 0.03
31COMMENT_BLOCK = '''#
32# List of boards
33#   Automatically generated by %s: don't edit
34#
35# Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers
36
37''' % __file__
38
39### helper functions ###
40def try_remove(f):
41    """Remove a file ignoring 'No such file or directory' error."""
42    try:
43        os.remove(f)
44    except OSError as exception:
45        # Ignore 'No such file or directory' error
46        if exception.errno != errno.ENOENT:
47            raise
48
49def check_top_directory():
50    """Exit if we are not at the top of source directory."""
51    for f in ('README', 'Licenses'):
52        if not os.path.exists(f):
53            sys.exit('Please run at the top of source directory.')
54
55def output_is_new(output):
56    """Check if the output file is up to date.
57
58    Returns:
59      True if the given output file exists and is newer than any of
60      *_defconfig, MAINTAINERS and Kconfig*.  False otherwise.
61    """
62    try:
63        ctime = os.path.getctime(output)
64    except OSError as exception:
65        if exception.errno == errno.ENOENT:
66            # return False on 'No such file or directory' error
67            return False
68        else:
69            raise
70
71    for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
72        for filename in fnmatch.filter(filenames, '*_defconfig'):
73            if fnmatch.fnmatch(filename, '.*'):
74                continue
75            filepath = os.path.join(dirpath, filename)
76            if ctime < os.path.getctime(filepath):
77                return False
78
79    for (dirpath, dirnames, filenames) in os.walk('.'):
80        for filename in filenames:
81            if (fnmatch.fnmatch(filename, '*~') or
82                not fnmatch.fnmatch(filename, 'Kconfig*') and
83                not filename == 'MAINTAINERS'):
84                continue
85            filepath = os.path.join(dirpath, filename)
86            if ctime < os.path.getctime(filepath):
87                return False
88
89    # Detect a board that has been removed since the current board database
90    # was generated
91    with open(output, encoding="utf-8") as f:
92        for line in f:
93            if line[0] == '#' or line == '\n':
94                continue
95            defconfig = line.split()[6] + '_defconfig'
96            if not os.path.exists(os.path.join(CONFIG_DIR, defconfig)):
97                return False
98
99    return True
100
101### classes ###
102class KconfigScanner:
103
104    """Kconfig scanner."""
105
106    ### constant variable only used in this class ###
107    _SYMBOL_TABLE = {
108        'arch' : 'SYS_ARCH',
109        'cpu' : 'SYS_CPU',
110        'soc' : 'SYS_SOC',
111        'vendor' : 'SYS_VENDOR',
112        'board' : 'SYS_BOARD',
113        'config' : 'SYS_CONFIG_NAME',
114        'options' : 'SYS_EXTRA_OPTIONS'
115    }
116
117    def __init__(self):
118        """Scan all the Kconfig files and create a Kconfig object."""
119        # Define environment variables referenced from Kconfig
120        os.environ['srctree'] = os.getcwd()
121        os.environ['UBOOTVERSION'] = 'dummy'
122        os.environ['KCONFIG_OBJDIR'] = ''
123        self._conf = kconfiglib.Kconfig(warn=False)
124
125    def __del__(self):
126        """Delete a leftover temporary file before exit.
127
128        The scan() method of this class creates a temporay file and deletes
129        it on success.  If scan() method throws an exception on the way,
130        the temporary file might be left over.  In that case, it should be
131        deleted in this destructor.
132        """
133        if hasattr(self, '_tmpfile') and self._tmpfile:
134            try_remove(self._tmpfile)
135
136    def scan(self, defconfig):
137        """Load a defconfig file to obtain board parameters.
138
139        Arguments:
140          defconfig: path to the defconfig file to be processed
141
142        Returns:
143          A dictionary of board parameters.  It has a form of:
144          {
145              'arch': <arch_name>,
146              'cpu': <cpu_name>,
147              'soc': <soc_name>,
148              'vendor': <vendor_name>,
149              'board': <board_name>,
150              'target': <target_name>,
151              'config': <config_header_name>,
152              'options': <extra_options>
153          }
154        """
155        # strip special prefixes and save it in a temporary file
156        fd, self._tmpfile = tempfile.mkstemp()
157        with os.fdopen(fd, 'w') as f:
158            for line in open(defconfig):
159                colon = line.find(':CONFIG_')
160                if colon == -1:
161                    f.write(line)
162                else:
163                    f.write(line[colon + 1:])
164
165        self._conf.load_config(self._tmpfile)
166        try_remove(self._tmpfile)
167        self._tmpfile = None
168
169        params = {}
170
171        # Get the value of CONFIG_SYS_ARCH, CONFIG_SYS_CPU, ... etc.
172        # Set '-' if the value is empty.
173        for key, symbol in list(self._SYMBOL_TABLE.items()):
174            value = self._conf.syms.get(symbol).str_value
175            if value:
176                params[key] = value
177            else:
178                params[key] = '-'
179
180        defconfig = os.path.basename(defconfig)
181        params['target'], match, rear = defconfig.partition('_defconfig')
182        assert match and not rear, '%s : invalid defconfig' % defconfig
183
184        # fix-up for aarch64
185        if params['arch'] == 'arm' and params['cpu'] == 'armv8':
186            params['arch'] = 'aarch64'
187
188        # fix-up options field. It should have the form:
189        # <config name>[:comma separated config options]
190        if params['options'] != '-':
191            params['options'] = params['config'] + ':' + \
192                                params['options'].replace(r'\"', '"')
193        elif params['config'] != params['target']:
194            params['options'] = params['config']
195
196        return params
197
198def scan_defconfigs_for_multiprocess(queue, defconfigs):
199    """Scan defconfig files and queue their board parameters
200
201    This function is intended to be passed to
202    multiprocessing.Process() constructor.
203
204    Arguments:
205      queue: An instance of multiprocessing.Queue().
206             The resulting board parameters are written into it.
207      defconfigs: A sequence of defconfig files to be scanned.
208    """
209    kconf_scanner = KconfigScanner()
210    for defconfig in defconfigs:
211        queue.put(kconf_scanner.scan(defconfig))
212
213def read_queues(queues, params_list):
214    """Read the queues and append the data to the paramers list"""
215    for q in queues:
216        while not q.empty():
217            params_list.append(q.get())
218
219def scan_defconfigs(jobs=1):
220    """Collect board parameters for all defconfig files.
221
222    This function invokes multiple processes for faster processing.
223
224    Arguments:
225      jobs: The number of jobs to run simultaneously
226    """
227    all_defconfigs = []
228    for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
229        for filename in fnmatch.filter(filenames, '*_defconfig'):
230            if fnmatch.fnmatch(filename, '.*'):
231                continue
232            all_defconfigs.append(os.path.join(dirpath, filename))
233
234    total_boards = len(all_defconfigs)
235    processes = []
236    queues = []
237    for i in range(jobs):
238        defconfigs = all_defconfigs[total_boards * i // jobs :
239                                    total_boards * (i + 1) // jobs]
240        q = multiprocessing.Queue(maxsize=-1)
241        p = multiprocessing.Process(target=scan_defconfigs_for_multiprocess,
242                                    args=(q, defconfigs))
243        p.start()
244        processes.append(p)
245        queues.append(q)
246
247    # The resulting data should be accumulated to this list
248    params_list = []
249
250    # Data in the queues should be retrieved preriodically.
251    # Otherwise, the queues would become full and subprocesses would get stuck.
252    while any([p.is_alive() for p in processes]):
253        read_queues(queues, params_list)
254        # sleep for a while until the queues are filled
255        time.sleep(SLEEP_TIME)
256
257    # Joining subprocesses just in case
258    # (All subprocesses should already have been finished)
259    for p in processes:
260        p.join()
261
262    # retrieve leftover data
263    read_queues(queues, params_list)
264
265    return params_list
266
267class MaintainersDatabase:
268
269    """The database of board status and maintainers."""
270
271    def __init__(self):
272        """Create an empty database."""
273        self.database = {}
274
275    def get_status(self, target):
276        """Return the status of the given board.
277
278        The board status is generally either 'Active' or 'Orphan'.
279        Display a warning message and return '-' if status information
280        is not found.
281
282        Returns:
283          'Active', 'Orphan' or '-'.
284        """
285        if not target in self.database:
286            print("WARNING: no status info for '%s'" % target, file=sys.stderr)
287            return '-'
288
289        tmp = self.database[target][0]
290        if tmp.startswith('Maintained'):
291            return 'Active'
292        elif tmp.startswith('Supported'):
293            return 'Active'
294        elif tmp.startswith('Orphan'):
295            return 'Orphan'
296        else:
297            print(("WARNING: %s: unknown status for '%s'" %
298                                  (tmp, target)), file=sys.stderr)
299            return '-'
300
301    def get_maintainers(self, target):
302        """Return the maintainers of the given board.
303
304        Returns:
305          Maintainers of the board.  If the board has two or more maintainers,
306          they are separated with colons.
307        """
308        if not target in self.database:
309            print("WARNING: no maintainers for '%s'" % target, file=sys.stderr)
310            return ''
311
312        return ':'.join(self.database[target][1])
313
314    def parse_file(self, file):
315        """Parse a MAINTAINERS file.
316
317        Parse a MAINTAINERS file and accumulates board status and
318        maintainers information.
319
320        Arguments:
321          file: MAINTAINERS file to be parsed
322        """
323        targets = []
324        maintainers = []
325        status = '-'
326        for line in open(file, encoding="utf-8"):
327            # Check also commented maintainers
328            if line[:3] == '#M:':
329                line = line[1:]
330            tag, rest = line[:2], line[2:].strip()
331            if tag == 'M:':
332                maintainers.append(rest)
333            elif tag == 'F:':
334                # expand wildcard and filter by 'configs/*_defconfig'
335                for f in glob.glob(rest):
336                    front, match, rear = f.partition('configs/')
337                    if not front and match:
338                        front, match, rear = rear.rpartition('_defconfig')
339                        if match and not rear:
340                            targets.append(front)
341            elif tag == 'S:':
342                status = rest
343            elif line == '\n':
344                for target in targets:
345                    self.database[target] = (status, maintainers)
346                targets = []
347                maintainers = []
348                status = '-'
349        if targets:
350            for target in targets:
351                self.database[target] = (status, maintainers)
352
353def insert_maintainers_info(params_list):
354    """Add Status and Maintainers information to the board parameters list.
355
356    Arguments:
357      params_list: A list of the board parameters
358    """
359    database = MaintainersDatabase()
360    for (dirpath, dirnames, filenames) in os.walk('.'):
361        if 'MAINTAINERS' in filenames:
362            database.parse_file(os.path.join(dirpath, 'MAINTAINERS'))
363
364    for i, params in enumerate(params_list):
365        target = params['target']
366        params['status'] = database.get_status(target)
367        params['maintainers'] = database.get_maintainers(target)
368        params_list[i] = params
369
370def format_and_output(params_list, output):
371    """Write board parameters into a file.
372
373    Columnate the board parameters, sort lines alphabetically,
374    and then write them to a file.
375
376    Arguments:
377      params_list: The list of board parameters
378      output: The path to the output file
379    """
380    FIELDS = ('status', 'arch', 'cpu', 'soc', 'vendor', 'board', 'target',
381              'options', 'maintainers')
382
383    # First, decide the width of each column
384    max_length = dict([ (f, 0) for f in FIELDS])
385    for params in params_list:
386        for f in FIELDS:
387            max_length[f] = max(max_length[f], len(params[f]))
388
389    output_lines = []
390    for params in params_list:
391        line = ''
392        for f in FIELDS:
393            # insert two spaces between fields like column -t would
394            line += '  ' + params[f].ljust(max_length[f])
395        output_lines.append(line.strip())
396
397    # ignore case when sorting
398    output_lines.sort(key=str.lower)
399
400    with open(output, 'w', encoding="utf-8") as f:
401        f.write(COMMENT_BLOCK + '\n'.join(output_lines) + '\n')
402
403def gen_boards_cfg(output, jobs=1, force=False, quiet=False):
404    """Generate a board database file.
405
406    Arguments:
407      output: The name of the output file
408      jobs: The number of jobs to run simultaneously
409      force: Force to generate the output even if it is new
410      quiet: True to avoid printing a message if nothing needs doing
411    """
412    check_top_directory()
413
414    if not force and output_is_new(output):
415        if not quiet:
416            print("%s is up to date. Nothing to do." % output)
417        sys.exit(0)
418
419    params_list = scan_defconfigs(jobs)
420    insert_maintainers_info(params_list)
421    format_and_output(params_list, output)
422
423def main():
424    try:
425        cpu_count = multiprocessing.cpu_count()
426    except NotImplementedError:
427        cpu_count = 1
428
429    parser = optparse.OptionParser()
430    # Add options here
431    parser.add_option('-f', '--force', action="store_true", default=False,
432                      help='regenerate the output even if it is new')
433    parser.add_option('-j', '--jobs', type='int', default=cpu_count,
434                      help='the number of jobs to run simultaneously')
435    parser.add_option('-o', '--output', default=OUTPUT_FILE,
436                      help='output file [default=%s]' % OUTPUT_FILE)
437    parser.add_option('-q', '--quiet', action="store_true", help='run silently')
438    (options, args) = parser.parse_args()
439
440    gen_boards_cfg(options.output, jobs=options.jobs, force=options.force,
441                   quiet=options.quiet)
442
443if __name__ == '__main__':
444    main()
445