1import argparse
2import glob
3import HTMLParser
4import logging
5import os
6import re
7import sys
8import urllib2
9
10
11# Import compare-locales parser from parent folder.
12script_path = os.path.dirname(os.path.realpath(__file__))
13compare_locales_path = os.path.join(script_path, '../../../../third_party/python/compare-locales')
14sys.path.insert(0, compare_locales_path)
15from compare_locales import parser
16
17
18# Configure logging format and level
19logging.basicConfig(format='  [%(levelname)s] %(message)s', level=logging.INFO)
20
21
22# License header to use when creating new properties files.
23DEFAULT_HEADER = ('# This Source Code Form is subject to the terms of the '
24                  'Mozilla Public\n# License, v. 2.0. If a copy of the MPL '
25                  'was not distributed with this\n# file, You can obtain '
26                  'one at http://mozilla.org/MPL/2.0/.\n')
27
28
29# Base url to retrieve properties files on central, that will be parsed for
30# localization notes.
31CENTRAL_BASE_URL = ('https://hg.mozilla.org/'
32                    'mozilla-central/raw-file/tip/'
33                    'devtools/client/locales/en-US/')
34
35
36# HTML parser to translate HTML entities in dtd files.
37HTML_PARSER = HTMLParser.HTMLParser()
38
39# Cache to store properties files retrieved over the network.
40central_prop_cache = {}
41
42# Cache the parsed entities from the existing DTD files.
43dtd_entities_cache = {}
44
45
46# Retrieve the content of the current version of a properties file for the
47# provided filename, from devtools/client on mozilla central. Will return an
48# empty array if the file can't be retrieved or read.
49def get_central_prop_content(prop_filename):
50    if prop_filename in central_prop_cache:
51        return central_prop_cache[prop_filename]
52
53    url = CENTRAL_BASE_URL + prop_filename
54    logging.info('loading localization file from central: {%s}' % url)
55
56    try:
57        central_prop_cache[prop_filename] = urllib2.urlopen(url).readlines()
58    except:
59        logging.error('failed to load properties file from central: {%s}'
60                      % url)
61        central_prop_cache[prop_filename] = []
62
63    return central_prop_cache[prop_filename]
64
65
66# Retrieve the current en-US localization notes for the provided prop_name.
67def get_localization_note(prop_name, prop_filename):
68    prop_content = get_central_prop_content(prop_filename)
69
70    comment_buffer = []
71    for i, line in enumerate(prop_content):
72        # Remove line breaks.
73        line = line.strip('\n').strip('\r')
74
75        if line.startswith('#'):
76            # Comment line, add to the current comment buffer.
77            comment_buffer.append(line)
78        elif re.search('(^|\n)' + re.escape(prop_name) + '\s*=', line):
79            # Property found, the current comment buffer is the localization
80            # note.
81            break;
82        else:
83            # No match, not a comment, reinitialize the comment buffer.
84            comment_buffer = []
85
86    return '\n'.join(comment_buffer)
87
88
89# Retrieve the parsed DTD entities for a provided path. Results are cached by
90# dtd path.
91def get_dtd_entities(dtd_path):
92    if dtd_path in dtd_entities_cache:
93        return dtd_entities_cache[dtd_path]
94
95    dtd_parser = parser.getParser('.dtd')
96    dtd_parser.readFile(dtd_path)
97    dtd_entities_cache[dtd_path] = dtd_parser.parse()
98    return dtd_entities_cache[dtd_path]
99
100
101# Extract the value of an entity in a dtd file.
102def get_translation_from_dtd(dtd_path, entity_name):
103    entities, map = get_dtd_entities(dtd_path)
104    if entity_name not in map:
105        # Bail out if translation is missing.
106        return
107
108    key = map[entity_name]
109    entity = entities[key]
110    translation = HTML_PARSER.unescape(entity.val)
111    return translation.encode('utf-8')
112
113
114# Extract the header and file wide comments for the provided properties file
115# filename.
116def get_properties_header(prop_filename):
117    prop_content = get_central_prop_content(prop_filename)
118
119    # if the file content is empty, return the default license header.
120    if len(prop_content) == 0:
121        return DEFAULT_HEADER
122
123    header_buffer = []
124    for i, line in enumerate(prop_content):
125        # remove line breaks.
126        line = line.strip('\n').strip('\r')
127
128        # regexp matching keys extracted form parser.py.
129        is_entity_line = re.search('^(\s*)'
130                                   '((?:[#!].*?\n\s*)*)'
131                                   '([^#!\s\n][^=:\n]*?)\s*[:=][ \t]*', line)
132        is_loc_note = re.search('^(\s*)'
133                                '\#\s*LOCALIZATION NOTE\s*\([^)]+\)', line)
134        if is_entity_line or is_loc_note:
135            # header finished, break the loop.
136            break
137        else:
138            # header line, add to the current buffer.
139            header_buffer.append(line)
140
141    # concatenate the current buffer and return.
142    return '\n'.join(header_buffer)
143
144
145# Create a new properties file at the provided path.
146def create_properties_file(prop_path):
147    logging.info('creating new *.properties file: {%s}' % prop_path)
148
149    prop_filename = os.path.basename(prop_path)
150    header = get_properties_header(prop_filename)
151
152    prop_file = open(prop_path, 'w+')
153    prop_file.write(header)
154    prop_file.close()
155
156
157# Migrate a single string entry for a dtd to a properties file.
158def migrate_string(dtd_path, prop_path, dtd_name, prop_name):
159    if not os.path.isfile(dtd_path):
160        logging.error('dtd file can not be found at: {%s}' % dtd_path)
161        return
162
163    translation = get_translation_from_dtd(dtd_path, dtd_name)
164    if not translation:
165        logging.error('translation could not be found for: {%s} in {%s}'
166                      % (dtd_name, dtd_path))
167        return
168
169    # Create properties file if missing.
170    if not os.path.isfile(prop_path):
171        create_properties_file(prop_path)
172
173    if not os.path.isfile(prop_path):
174        logging.error('could not create new properties file at: {%s}'
175                      % prop_path)
176        return
177
178    prop_line = prop_name + '=' + translation + '\n'
179
180    # Skip the string if it already exists in the destination file.
181    prop_file_content = open(prop_path, 'r').read()
182    if prop_line in prop_file_content:
183        logging.warning('string already migrated, skipping: {%s}' % prop_name)
184        return
185
186    # Skip the string and log an error if an existing entry is found, but with
187    # a different value.
188    if re.search('(^|\n)' + re.escape(prop_name) + '\s*=', prop_file_content):
189        logging.error('existing string found, skipping: {%s}' % prop_name)
190        return
191
192    prop_filename = os.path.basename(prop_path)
193    logging.info('migrating {%s} in {%s}' % (prop_name, prop_filename))
194    with open(prop_path, 'a') as prop_file:
195        localization_note = get_localization_note(prop_name, prop_filename)
196        if len(localization_note):
197            prop_file.write('\n' + localization_note)
198        else:
199            logging.warning('localization notes could not be found for: {%s}'
200                            % prop_name)
201        prop_file.write('\n' + prop_line)
202
203
204# Apply the migration instructions in the provided configuration file.
205def migrate_conf(conf_path, l10n_path):
206    f = open(conf_path, 'r')
207    lines = f.readlines()
208    f.close()
209
210    for i, line in enumerate(lines):
211        # Remove line breaks.
212        line = line.strip('\n').strip('\r')
213
214        # Skip invalid lines.
215        if ' = ' not in line:
216            continue
217
218        # Expected syntax: ${prop_path}:${prop_name} = ${dtd_path}:${dtd_name}.
219        prop_info, dtd_info = line.split(' = ')
220        prop_path, prop_name = prop_info.split(':')
221        dtd_path, dtd_name = dtd_info.split(':')
222
223        dtd_path = os.path.join(l10n_path, dtd_path)
224        prop_path = os.path.join(l10n_path, prop_path)
225
226        migrate_string(dtd_path, prop_path, dtd_name, prop_name)
227
228
229def main():
230    # Read command line arguments.
231    arg_parser = argparse.ArgumentParser(
232            description='Migrate devtools localized strings.')
233    arg_parser.add_argument('path', type=str, help='path to l10n repository')
234    arg_parser.add_argument('-c', '--config', type=str,
235                            help='path to configuration file or folder')
236    args = arg_parser.parse_args()
237
238    # Retrieve path to devtools localization files in l10n repository.
239    devtools_l10n_path = os.path.join(args.path, 'devtools/client/')
240    if not os.path.exists(devtools_l10n_path):
241        logging.error('l10n path is invalid: {%s}' % devtools_l10n_path)
242        exit()
243    logging.info('l10n path is valid: {%s}' % devtools_l10n_path)
244
245    # Retrieve configuration files to apply.
246    if os.path.isdir(args.config):
247        conf_files = glob.glob(args.config + '*')
248    elif os.path.isfile(args.config):
249        conf_files = [args.config]
250    else:
251        logging.error('config path is invalid: {%s}' % args.config)
252        exit()
253
254    # Perform migration for each configuration file.
255    for conf_file in conf_files:
256        logging.info('performing migration for config file: {%s}' % conf_file)
257        migrate_conf(conf_file, devtools_l10n_path)
258
259
260if __name__ == '__main__':
261    main()
262