1import argparse 2import glob 3import HTMLParser 4import logging 5import os 6import re 7import sys 8import urllib2 9 10 11# Import compare-locales parser from parent folder. 12script_path = os.path.dirname(os.path.realpath(__file__)) 13compare_locales_path = os.path.join(script_path, '../../../../third_party/python/compare-locales') 14sys.path.insert(0, compare_locales_path) 15from compare_locales import parser 16 17 18# Configure logging format and level 19logging.basicConfig(format=' [%(levelname)s] %(message)s', level=logging.INFO) 20 21 22# License header to use when creating new properties files. 23DEFAULT_HEADER = ('# This Source Code Form is subject to the terms of the ' 24 'Mozilla Public\n# License, v. 2.0. If a copy of the MPL ' 25 'was not distributed with this\n# file, You can obtain ' 26 'one at http://mozilla.org/MPL/2.0/.\n') 27 28 29# Base url to retrieve properties files on central, that will be parsed for 30# localization notes. 31CENTRAL_BASE_URL = ('https://hg.mozilla.org/' 32 'mozilla-central/raw-file/tip/' 33 'devtools/client/locales/en-US/') 34 35 36# HTML parser to translate HTML entities in dtd files. 37HTML_PARSER = HTMLParser.HTMLParser() 38 39# Cache to store properties files retrieved over the network. 40central_prop_cache = {} 41 42# Cache the parsed entities from the existing DTD files. 43dtd_entities_cache = {} 44 45 46# Retrieve the content of the current version of a properties file for the 47# provided filename, from devtools/client on mozilla central. Will return an 48# empty array if the file can't be retrieved or read. 49def get_central_prop_content(prop_filename): 50 if prop_filename in central_prop_cache: 51 return central_prop_cache[prop_filename] 52 53 url = CENTRAL_BASE_URL + prop_filename 54 logging.info('loading localization file from central: {%s}' % url) 55 56 try: 57 central_prop_cache[prop_filename] = urllib2.urlopen(url).readlines() 58 except: 59 logging.error('failed to load properties file from central: {%s}' 60 % url) 61 central_prop_cache[prop_filename] = [] 62 63 return central_prop_cache[prop_filename] 64 65 66# Retrieve the current en-US localization notes for the provided prop_name. 67def get_localization_note(prop_name, prop_filename): 68 prop_content = get_central_prop_content(prop_filename) 69 70 comment_buffer = [] 71 for i, line in enumerate(prop_content): 72 # Remove line breaks. 73 line = line.strip('\n').strip('\r') 74 75 if line.startswith('#'): 76 # Comment line, add to the current comment buffer. 77 comment_buffer.append(line) 78 elif re.search('(^|\n)' + re.escape(prop_name) + '\s*=', line): 79 # Property found, the current comment buffer is the localization 80 # note. 81 break; 82 else: 83 # No match, not a comment, reinitialize the comment buffer. 84 comment_buffer = [] 85 86 return '\n'.join(comment_buffer) 87 88 89# Retrieve the parsed DTD entities for a provided path. Results are cached by 90# dtd path. 91def get_dtd_entities(dtd_path): 92 if dtd_path in dtd_entities_cache: 93 return dtd_entities_cache[dtd_path] 94 95 dtd_parser = parser.getParser('.dtd') 96 dtd_parser.readFile(dtd_path) 97 dtd_entities_cache[dtd_path] = dtd_parser.parse() 98 return dtd_entities_cache[dtd_path] 99 100 101# Extract the value of an entity in a dtd file. 102def get_translation_from_dtd(dtd_path, entity_name): 103 entities, map = get_dtd_entities(dtd_path) 104 if entity_name not in map: 105 # Bail out if translation is missing. 106 return 107 108 key = map[entity_name] 109 entity = entities[key] 110 translation = HTML_PARSER.unescape(entity.val) 111 return translation.encode('utf-8') 112 113 114# Extract the header and file wide comments for the provided properties file 115# filename. 116def get_properties_header(prop_filename): 117 prop_content = get_central_prop_content(prop_filename) 118 119 # if the file content is empty, return the default license header. 120 if len(prop_content) == 0: 121 return DEFAULT_HEADER 122 123 header_buffer = [] 124 for i, line in enumerate(prop_content): 125 # remove line breaks. 126 line = line.strip('\n').strip('\r') 127 128 # regexp matching keys extracted form parser.py. 129 is_entity_line = re.search('^(\s*)' 130 '((?:[#!].*?\n\s*)*)' 131 '([^#!\s\n][^=:\n]*?)\s*[:=][ \t]*', line) 132 is_loc_note = re.search('^(\s*)' 133 '\#\s*LOCALIZATION NOTE\s*\([^)]+\)', line) 134 if is_entity_line or is_loc_note: 135 # header finished, break the loop. 136 break 137 else: 138 # header line, add to the current buffer. 139 header_buffer.append(line) 140 141 # concatenate the current buffer and return. 142 return '\n'.join(header_buffer) 143 144 145# Create a new properties file at the provided path. 146def create_properties_file(prop_path): 147 logging.info('creating new *.properties file: {%s}' % prop_path) 148 149 prop_filename = os.path.basename(prop_path) 150 header = get_properties_header(prop_filename) 151 152 prop_file = open(prop_path, 'w+') 153 prop_file.write(header) 154 prop_file.close() 155 156 157# Migrate a single string entry for a dtd to a properties file. 158def migrate_string(dtd_path, prop_path, dtd_name, prop_name): 159 if not os.path.isfile(dtd_path): 160 logging.error('dtd file can not be found at: {%s}' % dtd_path) 161 return 162 163 translation = get_translation_from_dtd(dtd_path, dtd_name) 164 if not translation: 165 logging.error('translation could not be found for: {%s} in {%s}' 166 % (dtd_name, dtd_path)) 167 return 168 169 # Create properties file if missing. 170 if not os.path.isfile(prop_path): 171 create_properties_file(prop_path) 172 173 if not os.path.isfile(prop_path): 174 logging.error('could not create new properties file at: {%s}' 175 % prop_path) 176 return 177 178 prop_line = prop_name + '=' + translation + '\n' 179 180 # Skip the string if it already exists in the destination file. 181 prop_file_content = open(prop_path, 'r').read() 182 if prop_line in prop_file_content: 183 logging.warning('string already migrated, skipping: {%s}' % prop_name) 184 return 185 186 # Skip the string and log an error if an existing entry is found, but with 187 # a different value. 188 if re.search('(^|\n)' + re.escape(prop_name) + '\s*=', prop_file_content): 189 logging.error('existing string found, skipping: {%s}' % prop_name) 190 return 191 192 prop_filename = os.path.basename(prop_path) 193 logging.info('migrating {%s} in {%s}' % (prop_name, prop_filename)) 194 with open(prop_path, 'a') as prop_file: 195 localization_note = get_localization_note(prop_name, prop_filename) 196 if len(localization_note): 197 prop_file.write('\n' + localization_note) 198 else: 199 logging.warning('localization notes could not be found for: {%s}' 200 % prop_name) 201 prop_file.write('\n' + prop_line) 202 203 204# Apply the migration instructions in the provided configuration file. 205def migrate_conf(conf_path, l10n_path): 206 f = open(conf_path, 'r') 207 lines = f.readlines() 208 f.close() 209 210 for i, line in enumerate(lines): 211 # Remove line breaks. 212 line = line.strip('\n').strip('\r') 213 214 # Skip invalid lines. 215 if ' = ' not in line: 216 continue 217 218 # Expected syntax: ${prop_path}:${prop_name} = ${dtd_path}:${dtd_name}. 219 prop_info, dtd_info = line.split(' = ') 220 prop_path, prop_name = prop_info.split(':') 221 dtd_path, dtd_name = dtd_info.split(':') 222 223 dtd_path = os.path.join(l10n_path, dtd_path) 224 prop_path = os.path.join(l10n_path, prop_path) 225 226 migrate_string(dtd_path, prop_path, dtd_name, prop_name) 227 228 229def main(): 230 # Read command line arguments. 231 arg_parser = argparse.ArgumentParser( 232 description='Migrate devtools localized strings.') 233 arg_parser.add_argument('path', type=str, help='path to l10n repository') 234 arg_parser.add_argument('-c', '--config', type=str, 235 help='path to configuration file or folder') 236 args = arg_parser.parse_args() 237 238 # Retrieve path to devtools localization files in l10n repository. 239 devtools_l10n_path = os.path.join(args.path, 'devtools/client/') 240 if not os.path.exists(devtools_l10n_path): 241 logging.error('l10n path is invalid: {%s}' % devtools_l10n_path) 242 exit() 243 logging.info('l10n path is valid: {%s}' % devtools_l10n_path) 244 245 # Retrieve configuration files to apply. 246 if os.path.isdir(args.config): 247 conf_files = glob.glob(args.config + '*') 248 elif os.path.isfile(args.config): 249 conf_files = [args.config] 250 else: 251 logging.error('config path is invalid: {%s}' % args.config) 252 exit() 253 254 # Perform migration for each configuration file. 255 for conf_file in conf_files: 256 logging.info('performing migration for config file: {%s}' % conf_file) 257 migrate_conf(conf_file, devtools_l10n_path) 258 259 260if __name__ == '__main__': 261 main() 262