1from __future__ import print_function 2import os 3from collections import defaultdict 4import fnmatch 5import argparse 6 7arguments = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter, 8 description='Parses files for content tags and creates and a doxygen suitable listing from the parse results.', 9 epilog='%(prog)s Parses FILES in the MATCH directory for TAGs.' 10 'The comment lines may be in either # or /// form in a file,' 11 ' but only one form throughout a file. The results of the parse are' 12 ' formatted for a doxygen ".dox" file.') 13arguments.add_argument('--match', default='./', 14 help='Root directory to search for files to parse.') 15arguments.add_argument('-f', '--files', metavar='MASK', nargs='*', default=['*.*'], 16 help='One or more file masks of files to parse.') 17arguments.add_argument('-x', '--exclude', metavar='MASK', nargs='*', default=[], 18 help='One or more file masks to exclude from results matching source_mask(s).') 19arguments.add_argument('--tag', nargs=2, default=['@content_tag{', '}'], help='Set of opening and closing' 20 ' strings which surround a tag name and precede the tag documentation.') 21arguments.add_argument('-o', '--output', default='output.txt', help='Full path to an output file.') 22arguments.add_argument('--link_source', metavar='CONTEXT_DIR BASE_URL LINE_PREFIX]', nargs='*', default=[], 23 help='Display linked source before each tag documentation. Requires arguments in order:' 24 ' CONTEXT_DIR BASE_URL LINE_PREFIX. File sources will link to' 25 ' {BASE_URL}{FILE}{LINE_PREFIX}{LINE_NUM}, where FILE is the relative path of CONTEXT_DIR.' 26 ' e.g. Given: FILE=a/b/c/xyz.txt LINE_NUM=4 CONTEXT_DIR=a/b BASE_URL=http://a.it LINE_PREFIX=?' 27 ' Link result will be: http://a.it/c/xyz.txt?4') 28arguments.add_argument('--dry_run', action='store_true', help='Does not write to OUTPUT, instead printing info' 29 ' to stdout.') 30 31args = vars(arguments.parse_args()) 32 33tag_open = args.get('tag')[0] 34tag_close = args.get('tag')[1] 35 36 37def add_doc_source(_file_name, _line_number, _content, _tags): 38 fn = _file_name.lstrip('./') 39 # index documentation by tag name storing the source file name, line number, and documentation string 40 _tags[_content[0]].add((fn, _line_number, _content[1].strip())) 41 42 43def parse_file(_parse_file, _tags): 44 with open(_parse_file, 'r', encoding='utf-8') as f: 45 match_line = 0 46 content = [] 47 special_comments_this_file = None 48 # Documentation may span multiple lines for a content tag. 49 # Matched lines are only stored in _tags once an end condition is met. 50 # End conditions are: a line not starting a comment, a new content tag, end of file 51 for raw_line in enumerate(f): 52 if raw_line[1].lstrip().startswith("#") and special_comments_this_file is not True: 53 if special_comments_this_file is None: 54 special_comments_this_file = False 55 if tag_open in raw_line[1]: 56 if match_line: 57 # in event focs scripts decide to stack one documentation on top of another 58 add_doc_source(_parse_file, match_line, content, _tags) 59 # store content and line for later addition 60 content = ''.join(raw_line[1].split(tag_open, 1)[1]).split(tag_close, 1) 61 content[1] = content[1].strip() 62 match_line = raw_line[0] + 1 63 elif match_line: 64 # not a new content tag, append to previous line description 65 content[1] += ' ' + raw_line[1].lstrip('# ') 66 elif raw_line[1].lstrip().startswith("/// ") and special_comments_this_file is not False: 67 if special_comments_this_file is None: 68 special_comments_this_file = True 69 content = ['# '] 70 if tag_open in raw_line[1]: 71 if match_line: 72 # in event focs scripts decide to stack one documentation on top of another 73 add_doc_source(_parse_file, match_line, content, _tags) 74 # store content and line for later addition 75 content = ''.join(raw_line[1].split(tag_open, 1)[1]).split(tag_close, 1) 76 content[1] = content[1].strip() 77 match_line = raw_line[0] + 1 78 elif match_line: 79 # not a new content tag, append to previous line description 80 content[1] += ' ' + raw_line[1].lstrip('/// ') 81 elif match_line: 82 # end of description, add a node for this source 83 add_doc_source(_parse_file, match_line, content, _tags) 84 match_line = 0 85 content = [] 86 # EOF 87 if match_line: 88 # in event a matched comment is the last line in a file 89 add_doc_source(_parse_file, match_line, content, _tags) 90 91 92def get_link(_file, _line_number): 93 link_args = args.get('link_source') 94 if len(link_args) != 3: 95 if link_args: 96 print('Invalid link_source arguments: ', link_args) 97 return '' 98 base_path = os.path.relpath(_file, link_args[0].lstrip('./')) 99 link = os.path.join(link_args[1], base_path) 100 link += str.format('{0}{1}', link_args[2], _line_number) 101 return str.format('[{0}]({1})', base_path, link) 102 103 104all_tags = defaultdict(set) 105 106# parse files for tags 107for _file_path, _, _files in os.walk(args.get('match')): 108 for file_name in enumerate(_files): 109 file_match = False 110 # if this file should be checked 111 for mask in args.get('files'): 112 if fnmatch.fnmatch(file_name[1], mask): 113 file_match = True 114 if file_match: 115 # check for exclusions 116 for mask in args.get('exclude'): 117 if fnmatch.fnmatch(file_name[1], mask): 118 file_match = False 119 if file_match: 120 parse_file(os.path.join(_file_path, file_name[1]), all_tags) 121 122output_buff = [] 123 124# create tags formatted for doxygen 125 126output_buff.append('/**\n') 127output_buff.append('@page content_tag_listing Content Definition Tag Listing\n') 128output_buff.append('This page is required to load parsed data into the documentation, the actual listing can be found @ref content_tags\n') 129for tag in sorted(all_tags): 130 output_buff.append('@content_tag{' + tag + '} ') 131 for source in all_tags[tag]: 132 description = str.format('{0} ', source[2]) 133 # prepend source and link if requested 134 link_str = get_link(source[0], source[1]) 135 if link_str: 136 link_str += " - " 137 output_buff.append(link_str + description) 138 output_buff.append('\n') 139output_buff.append('*/\n') 140 141if args.get('dry_run'): 142 source_count = 0 143 for tag in all_tags: 144 for value in all_tags[tag]: 145 source_count += 1 146 print("arguments:") 147 for arg in args: 148 print("\t", arg, ":", args[arg]) 149 print("\nFound {0} tags with {1} total sources".format(len(all_tags.keys()), source_count)) 150else: 151 abs_dir = os.path.dirname(os.path.abspath(args.get('output'))) 152 if not os.path.exists(abs_dir): 153 os.mkdir(abs_dir) 154 with open(args.get('output'), 'w') as output_file: 155 output_file.writelines(output_buff) 156