1from __future__ import print_function
2import os
3from collections import defaultdict
4import fnmatch
5import argparse
6
7arguments = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter,
8                                    description='Parses files for content tags and creates and a doxygen suitable listing from the parse results.',
9                                    epilog='%(prog)s Parses FILES in the MATCH directory for TAGs.'
10                                           'The comment lines may be in either # or /// form in a file,'
11                                           ' but only one form throughout a file.  The results of the parse are'
12                                           ' formatted for a doxygen ".dox" file.')
13arguments.add_argument('--match', default='./',
14                       help='Root directory to search for files to parse.')
15arguments.add_argument('-f', '--files', metavar='MASK', nargs='*', default=['*.*'],
16                       help='One or more file masks of files to parse.')
17arguments.add_argument('-x', '--exclude', metavar='MASK', nargs='*', default=[],
18                       help='One or more file masks to exclude from results matching source_mask(s).')
19arguments.add_argument('--tag', nargs=2, default=['@content_tag{', '}'], help='Set of opening and closing'
20                       ' strings which surround a tag name and precede the tag documentation.')
21arguments.add_argument('-o', '--output', default='output.txt', help='Full path to an output file.')
22arguments.add_argument('--link_source', metavar='CONTEXT_DIR BASE_URL LINE_PREFIX]', nargs='*', default=[],
23                       help='Display linked source before each tag documentation. Requires arguments in order:'
24                       ' CONTEXT_DIR BASE_URL LINE_PREFIX. File sources will link to'
25                       ' {BASE_URL}{FILE}{LINE_PREFIX}{LINE_NUM}, where FILE is the relative path of CONTEXT_DIR.'
26                       ' e.g. Given: FILE=a/b/c/xyz.txt LINE_NUM=4 CONTEXT_DIR=a/b BASE_URL=http://a.it LINE_PREFIX=?'
27                       ' Link result will be: http://a.it/c/xyz.txt?4')
28arguments.add_argument('--dry_run', action='store_true', help='Does not write to OUTPUT, instead printing info'
29                       ' to stdout.')
30
31args = vars(arguments.parse_args())
32
33tag_open = args.get('tag')[0]
34tag_close = args.get('tag')[1]
35
36
37def add_doc_source(_file_name, _line_number, _content, _tags):
38    fn = _file_name.lstrip('./')
39    # index documentation by tag name storing the source file name, line number, and documentation string
40    _tags[_content[0]].add((fn, _line_number, _content[1].strip()))
41
42
43def parse_file(_parse_file, _tags):
44    with open(_parse_file, 'r', encoding='utf-8') as f:
45        match_line = 0
46        content = []
47        special_comments_this_file = None
48        # Documentation may span multiple lines for a content tag.
49        # Matched lines are only stored in _tags once an end condition is met.
50        # End conditions are: a line not starting a comment, a new content tag, end of file
51        for raw_line in enumerate(f):
52            if raw_line[1].lstrip().startswith("#") and special_comments_this_file is not True:
53                if special_comments_this_file is None:
54                    special_comments_this_file = False
55                if tag_open in raw_line[1]:
56                    if match_line:
57                        # in event focs scripts decide to stack one documentation on top of another
58                        add_doc_source(_parse_file, match_line, content, _tags)
59                    # store content and line for later addition
60                    content = ''.join(raw_line[1].split(tag_open, 1)[1]).split(tag_close, 1)
61                    content[1] = content[1].strip()
62                    match_line = raw_line[0] + 1
63                elif match_line:
64                    # not a new content tag, append to previous line description
65                    content[1] += ' ' + raw_line[1].lstrip('# ')
66            elif raw_line[1].lstrip().startswith("/// ") and special_comments_this_file is not False:
67                if special_comments_this_file is None:
68                    special_comments_this_file = True
69                    content = ['# ']
70                if tag_open in raw_line[1]:
71                    if match_line:
72                        # in event focs scripts decide to stack one documentation on top of another
73                        add_doc_source(_parse_file, match_line, content, _tags)
74                    # store content and line for later addition
75                    content = ''.join(raw_line[1].split(tag_open, 1)[1]).split(tag_close, 1)
76                    content[1] = content[1].strip()
77                    match_line = raw_line[0] + 1
78                elif match_line:
79                    # not a new content tag, append to previous line description
80                    content[1] += ' ' + raw_line[1].lstrip('/// ')
81            elif match_line:
82                # end of description, add a node for this source
83                add_doc_source(_parse_file, match_line, content, _tags)
84                match_line = 0
85                content = []
86        # EOF
87        if match_line:
88            # in event a matched comment is the last line in a file
89            add_doc_source(_parse_file, match_line, content, _tags)
90
91
92def get_link(_file, _line_number):
93    link_args = args.get('link_source')
94    if len(link_args) != 3:
95        if link_args:
96            print('Invalid link_source arguments: ', link_args)
97        return ''
98    base_path = os.path.relpath(_file, link_args[0].lstrip('./'))
99    link = os.path.join(link_args[1], base_path)
100    link += str.format('{0}{1}', link_args[2], _line_number)
101    return str.format('[{0}]({1})', base_path, link)
102
103
104all_tags = defaultdict(set)
105
106# parse files for tags
107for _file_path, _, _files in os.walk(args.get('match')):
108    for file_name in enumerate(_files):
109        file_match = False
110        # if this file should be checked
111        for mask in args.get('files'):
112            if fnmatch.fnmatch(file_name[1], mask):
113                file_match = True
114        if file_match:
115            # check for exclusions
116            for mask in args.get('exclude'):
117                if fnmatch.fnmatch(file_name[1], mask):
118                    file_match = False
119            if file_match:
120                parse_file(os.path.join(_file_path, file_name[1]), all_tags)
121
122output_buff = []
123
124# create tags formatted for doxygen
125
126output_buff.append('/**\n')
127output_buff.append('@page content_tag_listing Content Definition Tag Listing\n')
128output_buff.append('This page is required to load parsed data into the documentation, the actual listing can be found @ref content_tags\n')
129for tag in sorted(all_tags):
130    output_buff.append('@content_tag{' + tag + '} ')
131    for source in all_tags[tag]:
132        description = str.format('{0}  ', source[2])
133        # prepend source and link if requested
134        link_str = get_link(source[0], source[1])
135        if link_str:
136            link_str += " - "
137        output_buff.append(link_str + description)
138    output_buff.append('\n')
139output_buff.append('*/\n')
140
141if args.get('dry_run'):
142    source_count = 0
143    for tag in all_tags:
144        for value in all_tags[tag]:
145            source_count += 1
146    print("arguments:")
147    for arg in args:
148        print("\t", arg, ":", args[arg])
149    print("\nFound {0} tags with {1} total sources".format(len(all_tags.keys()), source_count))
150else:
151    abs_dir = os.path.dirname(os.path.abspath(args.get('output')))
152    if not os.path.exists(abs_dir):
153        os.mkdir(abs_dir)
154    with open(args.get('output'), 'w') as output_file:
155        output_file.writelines(output_buff)
156