1# -*- coding: utf-8 -*- 2""" 3 babel.messages.frontend 4 ~~~~~~~~~~~~~~~~~~~~~~~ 5 6 Frontends for the message extraction functionality. 7 8 :copyright: (c) 2013-2021 by the Babel Team. 9 :license: BSD, see LICENSE for more details. 10""" 11from __future__ import print_function 12 13import logging 14import optparse 15import os 16import re 17import shutil 18import sys 19import tempfile 20from collections import OrderedDict 21from datetime import datetime 22from locale import getpreferredencoding 23 24from babel import __version__ as VERSION 25from babel import Locale, localedata 26from babel._compat import StringIO, string_types, text_type, PY2 27from babel.core import UnknownLocaleError 28from babel.messages.catalog import Catalog 29from babel.messages.extract import DEFAULT_KEYWORDS, DEFAULT_MAPPING, check_and_call_extract_file, extract_from_dir 30from babel.messages.mofile import write_mo 31from babel.messages.pofile import read_po, write_po 32from babel.util import LOCALTZ 33from distutils import log as distutils_log 34from distutils.cmd import Command as _Command 35from distutils.errors import DistutilsOptionError, DistutilsSetupError 36 37try: 38 from ConfigParser import RawConfigParser 39except ImportError: 40 from configparser import RawConfigParser 41 42 43po_file_read_mode = ('rU' if PY2 else 'r') 44 45 46def listify_value(arg, split=None): 47 """ 48 Make a list out of an argument. 49 50 Values from `distutils` argument parsing are always single strings; 51 values from `optparse` parsing may be lists of strings that may need 52 to be further split. 53 54 No matter the input, this function returns a flat list of whitespace-trimmed 55 strings, with `None` values filtered out. 56 57 >>> listify_value("foo bar") 58 ['foo', 'bar'] 59 >>> listify_value(["foo bar"]) 60 ['foo', 'bar'] 61 >>> listify_value([["foo"], "bar"]) 62 ['foo', 'bar'] 63 >>> listify_value([["foo"], ["bar", None, "foo"]]) 64 ['foo', 'bar', 'foo'] 65 >>> listify_value("foo, bar, quux", ",") 66 ['foo', 'bar', 'quux'] 67 68 :param arg: A string or a list of strings 69 :param split: The argument to pass to `str.split()`. 70 :return: 71 """ 72 out = [] 73 74 if not isinstance(arg, (list, tuple)): 75 arg = [arg] 76 77 for val in arg: 78 if val is None: 79 continue 80 if isinstance(val, (list, tuple)): 81 out.extend(listify_value(val, split=split)) 82 continue 83 out.extend(s.strip() for s in text_type(val).split(split)) 84 assert all(isinstance(val, string_types) for val in out) 85 return out 86 87 88class Command(_Command): 89 # This class is a small shim between Distutils commands and 90 # optparse option parsing in the frontend command line. 91 92 #: Option name to be input as `args` on the script command line. 93 as_args = None 94 95 #: Options which allow multiple values. 96 #: This is used by the `optparse` transmogrification code. 97 multiple_value_options = () 98 99 #: Options which are booleans. 100 #: This is used by the `optparse` transmogrification code. 101 # (This is actually used by distutils code too, but is never 102 # declared in the base class.) 103 boolean_options = () 104 105 #: Option aliases, to retain standalone command compatibility. 106 #: Distutils does not support option aliases, but optparse does. 107 #: This maps the distutils argument name to an iterable of aliases 108 #: that are usable with optparse. 109 option_aliases = {} 110 111 #: Choices for options that needed to be restricted to specific 112 #: list of choices. 113 option_choices = {} 114 115 #: Log object. To allow replacement in the script command line runner. 116 log = distutils_log 117 118 def __init__(self, dist=None): 119 # A less strict version of distutils' `__init__`. 120 self.distribution = dist 121 self.initialize_options() 122 self._dry_run = None 123 self.verbose = False 124 self.force = None 125 self.help = 0 126 self.finalized = 0 127 128 129class compile_catalog(Command): 130 """Catalog compilation command for use in ``setup.py`` scripts. 131 132 If correctly installed, this command is available to Setuptools-using 133 setup scripts automatically. For projects using plain old ``distutils``, 134 the command needs to be registered explicitly in ``setup.py``:: 135 136 from babel.messages.frontend import compile_catalog 137 138 setup( 139 ... 140 cmdclass = {'compile_catalog': compile_catalog} 141 ) 142 143 .. versionadded:: 0.9 144 """ 145 146 description = 'compile message catalogs to binary MO files' 147 user_options = [ 148 ('domain=', 'D', 149 "domains of PO files (space separated list, default 'messages')"), 150 ('directory=', 'd', 151 'path to base directory containing the catalogs'), 152 ('input-file=', 'i', 153 'name of the input file'), 154 ('output-file=', 'o', 155 "name of the output file (default " 156 "'<output_dir>/<locale>/LC_MESSAGES/<domain>.mo')"), 157 ('locale=', 'l', 158 'locale of the catalog to compile'), 159 ('use-fuzzy', 'f', 160 'also include fuzzy translations'), 161 ('statistics', None, 162 'print statistics about translations') 163 ] 164 boolean_options = ['use-fuzzy', 'statistics'] 165 166 def initialize_options(self): 167 self.domain = 'messages' 168 self.directory = None 169 self.input_file = None 170 self.output_file = None 171 self.locale = None 172 self.use_fuzzy = False 173 self.statistics = False 174 175 def finalize_options(self): 176 self.domain = listify_value(self.domain) 177 if not self.input_file and not self.directory: 178 raise DistutilsOptionError('you must specify either the input file ' 179 'or the base directory') 180 if not self.output_file and not self.directory: 181 raise DistutilsOptionError('you must specify either the output file ' 182 'or the base directory') 183 184 def run(self): 185 n_errors = 0 186 for domain in self.domain: 187 for catalog, errors in self._run_domain(domain).items(): 188 n_errors += len(errors) 189 if n_errors: 190 self.log.error('%d errors encountered.' % n_errors) 191 return (1 if n_errors else 0) 192 193 def _run_domain(self, domain): 194 po_files = [] 195 mo_files = [] 196 197 if not self.input_file: 198 if self.locale: 199 po_files.append((self.locale, 200 os.path.join(self.directory, self.locale, 201 'LC_MESSAGES', 202 domain + '.po'))) 203 mo_files.append(os.path.join(self.directory, self.locale, 204 'LC_MESSAGES', 205 domain + '.mo')) 206 else: 207 for locale in os.listdir(self.directory): 208 po_file = os.path.join(self.directory, locale, 209 'LC_MESSAGES', domain + '.po') 210 if os.path.exists(po_file): 211 po_files.append((locale, po_file)) 212 mo_files.append(os.path.join(self.directory, locale, 213 'LC_MESSAGES', 214 domain + '.mo')) 215 else: 216 po_files.append((self.locale, self.input_file)) 217 if self.output_file: 218 mo_files.append(self.output_file) 219 else: 220 mo_files.append(os.path.join(self.directory, self.locale, 221 'LC_MESSAGES', 222 domain + '.mo')) 223 224 if not po_files: 225 raise DistutilsOptionError('no message catalogs found') 226 227 catalogs_and_errors = {} 228 229 for idx, (locale, po_file) in enumerate(po_files): 230 mo_file = mo_files[idx] 231 with open(po_file, 'rb') as infile: 232 catalog = read_po(infile, locale) 233 234 if self.statistics: 235 translated = 0 236 for message in list(catalog)[1:]: 237 if message.string: 238 translated += 1 239 percentage = 0 240 if len(catalog): 241 percentage = translated * 100 // len(catalog) 242 self.log.info( 243 '%d of %d messages (%d%%) translated in %s', 244 translated, len(catalog), percentage, po_file 245 ) 246 247 if catalog.fuzzy and not self.use_fuzzy: 248 self.log.info('catalog %s is marked as fuzzy, skipping', po_file) 249 continue 250 251 catalogs_and_errors[catalog] = catalog_errors = list(catalog.check()) 252 for message, errors in catalog_errors: 253 for error in errors: 254 self.log.error( 255 'error: %s:%d: %s', po_file, message.lineno, error 256 ) 257 258 self.log.info('compiling catalog %s to %s', po_file, mo_file) 259 260 with open(mo_file, 'wb') as outfile: 261 write_mo(outfile, catalog, use_fuzzy=self.use_fuzzy) 262 263 return catalogs_and_errors 264 265 266class extract_messages(Command): 267 """Message extraction command for use in ``setup.py`` scripts. 268 269 If correctly installed, this command is available to Setuptools-using 270 setup scripts automatically. For projects using plain old ``distutils``, 271 the command needs to be registered explicitly in ``setup.py``:: 272 273 from babel.messages.frontend import extract_messages 274 275 setup( 276 ... 277 cmdclass = {'extract_messages': extract_messages} 278 ) 279 """ 280 281 description = 'extract localizable strings from the project code' 282 user_options = [ 283 ('charset=', None, 284 'charset to use in the output file (default "utf-8")'), 285 ('keywords=', 'k', 286 'space-separated list of keywords to look for in addition to the ' 287 'defaults (may be repeated multiple times)'), 288 ('no-default-keywords', None, 289 'do not include the default keywords'), 290 ('mapping-file=', 'F', 291 'path to the mapping configuration file'), 292 ('no-location', None, 293 'do not include location comments with filename and line number'), 294 ('add-location=', None, 295 'location lines format. If it is not given or "full", it generates ' 296 'the lines with both file name and line number. If it is "file", ' 297 'the line number part is omitted. If it is "never", it completely ' 298 'suppresses the lines (same as --no-location).'), 299 ('omit-header', None, 300 'do not include msgid "" entry in header'), 301 ('output-file=', 'o', 302 'name of the output file'), 303 ('width=', 'w', 304 'set output line width (default 76)'), 305 ('no-wrap', None, 306 'do not break long message lines, longer than the output line width, ' 307 'into several lines'), 308 ('sort-output', None, 309 'generate sorted output (default False)'), 310 ('sort-by-file', None, 311 'sort output by file location (default False)'), 312 ('msgid-bugs-address=', None, 313 'set report address for msgid'), 314 ('copyright-holder=', None, 315 'set copyright holder in output'), 316 ('project=', None, 317 'set project name in output'), 318 ('version=', None, 319 'set project version in output'), 320 ('add-comments=', 'c', 321 'place comment block with TAG (or those preceding keyword lines) in ' 322 'output file. Separate multiple TAGs with commas(,)'), # TODO: Support repetition of this argument 323 ('strip-comments', 's', 324 'strip the comment TAGs from the comments.'), 325 ('input-paths=', None, 326 'files or directories that should be scanned for messages. Separate multiple ' 327 'files or directories with commas(,)'), # TODO: Support repetition of this argument 328 ('input-dirs=', None, # TODO (3.x): Remove me. 329 'alias for input-paths (does allow files as well as directories).'), 330 ] 331 boolean_options = [ 332 'no-default-keywords', 'no-location', 'omit-header', 'no-wrap', 333 'sort-output', 'sort-by-file', 'strip-comments' 334 ] 335 as_args = 'input-paths' 336 multiple_value_options = ('add-comments', 'keywords') 337 option_aliases = { 338 'keywords': ('--keyword',), 339 'mapping-file': ('--mapping',), 340 'output-file': ('--output',), 341 'strip-comments': ('--strip-comment-tags',), 342 } 343 option_choices = { 344 'add-location': ('full', 'file', 'never',), 345 } 346 347 def initialize_options(self): 348 self.charset = 'utf-8' 349 self.keywords = None 350 self.no_default_keywords = False 351 self.mapping_file = None 352 self.no_location = False 353 self.add_location = None 354 self.omit_header = False 355 self.output_file = None 356 self.input_dirs = None 357 self.input_paths = None 358 self.width = None 359 self.no_wrap = False 360 self.sort_output = False 361 self.sort_by_file = False 362 self.msgid_bugs_address = None 363 self.copyright_holder = None 364 self.project = None 365 self.version = None 366 self.add_comments = None 367 self.strip_comments = False 368 self.include_lineno = True 369 370 def finalize_options(self): 371 if self.input_dirs: 372 if not self.input_paths: 373 self.input_paths = self.input_dirs 374 else: 375 raise DistutilsOptionError( 376 'input-dirs and input-paths are mutually exclusive' 377 ) 378 379 if self.no_default_keywords: 380 keywords = {} 381 else: 382 keywords = DEFAULT_KEYWORDS.copy() 383 384 keywords.update(parse_keywords(listify_value(self.keywords))) 385 386 self.keywords = keywords 387 388 if not self.keywords: 389 raise DistutilsOptionError('you must specify new keywords if you ' 390 'disable the default ones') 391 392 if not self.output_file: 393 raise DistutilsOptionError('no output file specified') 394 if self.no_wrap and self.width: 395 raise DistutilsOptionError("'--no-wrap' and '--width' are mutually " 396 "exclusive") 397 if not self.no_wrap and not self.width: 398 self.width = 76 399 elif self.width is not None: 400 self.width = int(self.width) 401 402 if self.sort_output and self.sort_by_file: 403 raise DistutilsOptionError("'--sort-output' and '--sort-by-file' " 404 "are mutually exclusive") 405 406 if self.input_paths: 407 if isinstance(self.input_paths, string_types): 408 self.input_paths = re.split(r',\s*', self.input_paths) 409 elif self.distribution is not None: 410 self.input_paths = dict.fromkeys([ 411 k.split('.', 1)[0] 412 for k in (self.distribution.packages or ()) 413 ]).keys() 414 else: 415 self.input_paths = [] 416 417 if not self.input_paths: 418 raise DistutilsOptionError("no input files or directories specified") 419 420 for path in self.input_paths: 421 if not os.path.exists(path): 422 raise DistutilsOptionError("Input path: %s does not exist" % path) 423 424 self.add_comments = listify_value(self.add_comments or (), ",") 425 426 if self.distribution: 427 if not self.project: 428 self.project = self.distribution.get_name() 429 if not self.version: 430 self.version = self.distribution.get_version() 431 432 if self.add_location == 'never': 433 self.no_location = True 434 elif self.add_location == 'file': 435 self.include_lineno = False 436 437 def run(self): 438 mappings = self._get_mappings() 439 with open(self.output_file, 'wb') as outfile: 440 catalog = Catalog(project=self.project, 441 version=self.version, 442 msgid_bugs_address=self.msgid_bugs_address, 443 copyright_holder=self.copyright_holder, 444 charset=self.charset) 445 446 for path, method_map, options_map in mappings: 447 def callback(filename, method, options): 448 if method == 'ignore': 449 return 450 451 # If we explicitly provide a full filepath, just use that. 452 # Otherwise, path will be the directory path and filename 453 # is the relative path from that dir to the file. 454 # So we can join those to get the full filepath. 455 if os.path.isfile(path): 456 filepath = path 457 else: 458 filepath = os.path.normpath(os.path.join(path, filename)) 459 460 optstr = '' 461 if options: 462 optstr = ' (%s)' % ', '.join(['%s="%s"' % (k, v) for 463 k, v in options.items()]) 464 self.log.info('extracting messages from %s%s', filepath, optstr) 465 466 if os.path.isfile(path): 467 current_dir = os.getcwd() 468 extracted = check_and_call_extract_file( 469 path, method_map, options_map, 470 callback, self.keywords, self.add_comments, 471 self.strip_comments, current_dir 472 ) 473 else: 474 extracted = extract_from_dir( 475 path, method_map, options_map, 476 keywords=self.keywords, 477 comment_tags=self.add_comments, 478 callback=callback, 479 strip_comment_tags=self.strip_comments 480 ) 481 for filename, lineno, message, comments, context in extracted: 482 if os.path.isfile(path): 483 filepath = filename # already normalized 484 else: 485 filepath = os.path.normpath(os.path.join(path, filename)) 486 487 catalog.add(message, None, [(filepath, lineno)], 488 auto_comments=comments, context=context) 489 490 self.log.info('writing PO template file to %s', self.output_file) 491 write_po(outfile, catalog, width=self.width, 492 no_location=self.no_location, 493 omit_header=self.omit_header, 494 sort_output=self.sort_output, 495 sort_by_file=self.sort_by_file, 496 include_lineno=self.include_lineno) 497 498 def _get_mappings(self): 499 mappings = [] 500 501 if self.mapping_file: 502 with open(self.mapping_file, po_file_read_mode) as fileobj: 503 method_map, options_map = parse_mapping(fileobj) 504 for path in self.input_paths: 505 mappings.append((path, method_map, options_map)) 506 507 elif getattr(self.distribution, 'message_extractors', None): 508 message_extractors = self.distribution.message_extractors 509 for path, mapping in message_extractors.items(): 510 if isinstance(mapping, string_types): 511 method_map, options_map = parse_mapping(StringIO(mapping)) 512 else: 513 method_map, options_map = [], {} 514 for pattern, method, options in mapping: 515 method_map.append((pattern, method)) 516 options_map[pattern] = options or {} 517 mappings.append((path, method_map, options_map)) 518 519 else: 520 for path in self.input_paths: 521 mappings.append((path, DEFAULT_MAPPING, {})) 522 523 return mappings 524 525 526def check_message_extractors(dist, name, value): 527 """Validate the ``message_extractors`` keyword argument to ``setup()``. 528 529 :param dist: the distutils/setuptools ``Distribution`` object 530 :param name: the name of the keyword argument (should always be 531 "message_extractors") 532 :param value: the value of the keyword argument 533 :raise `DistutilsSetupError`: if the value is not valid 534 """ 535 assert name == 'message_extractors' 536 if not isinstance(value, dict): 537 raise DistutilsSetupError('the value of the "message_extractors" ' 538 'parameter must be a dictionary') 539 540 541class init_catalog(Command): 542 """New catalog initialization command for use in ``setup.py`` scripts. 543 544 If correctly installed, this command is available to Setuptools-using 545 setup scripts automatically. For projects using plain old ``distutils``, 546 the command needs to be registered explicitly in ``setup.py``:: 547 548 from babel.messages.frontend import init_catalog 549 550 setup( 551 ... 552 cmdclass = {'init_catalog': init_catalog} 553 ) 554 """ 555 556 description = 'create a new catalog based on a POT file' 557 user_options = [ 558 ('domain=', 'D', 559 "domain of PO file (default 'messages')"), 560 ('input-file=', 'i', 561 'name of the input file'), 562 ('output-dir=', 'd', 563 'path to output directory'), 564 ('output-file=', 'o', 565 "name of the output file (default " 566 "'<output_dir>/<locale>/LC_MESSAGES/<domain>.po')"), 567 ('locale=', 'l', 568 'locale for the new localized catalog'), 569 ('width=', 'w', 570 'set output line width (default 76)'), 571 ('no-wrap', None, 572 'do not break long message lines, longer than the output line width, ' 573 'into several lines'), 574 ] 575 boolean_options = ['no-wrap'] 576 577 def initialize_options(self): 578 self.output_dir = None 579 self.output_file = None 580 self.input_file = None 581 self.locale = None 582 self.domain = 'messages' 583 self.no_wrap = False 584 self.width = None 585 586 def finalize_options(self): 587 if not self.input_file: 588 raise DistutilsOptionError('you must specify the input file') 589 590 if not self.locale: 591 raise DistutilsOptionError('you must provide a locale for the ' 592 'new catalog') 593 try: 594 self._locale = Locale.parse(self.locale) 595 except UnknownLocaleError as e: 596 raise DistutilsOptionError(e) 597 598 if not self.output_file and not self.output_dir: 599 raise DistutilsOptionError('you must specify the output directory') 600 if not self.output_file: 601 self.output_file = os.path.join(self.output_dir, self.locale, 602 'LC_MESSAGES', self.domain + '.po') 603 604 if not os.path.exists(os.path.dirname(self.output_file)): 605 os.makedirs(os.path.dirname(self.output_file)) 606 if self.no_wrap and self.width: 607 raise DistutilsOptionError("'--no-wrap' and '--width' are mutually " 608 "exclusive") 609 if not self.no_wrap and not self.width: 610 self.width = 76 611 elif self.width is not None: 612 self.width = int(self.width) 613 614 def run(self): 615 self.log.info( 616 'creating catalog %s based on %s', self.output_file, self.input_file 617 ) 618 619 with open(self.input_file, 'rb') as infile: 620 # Although reading from the catalog template, read_po must be fed 621 # the locale in order to correctly calculate plurals 622 catalog = read_po(infile, locale=self.locale) 623 624 catalog.locale = self._locale 625 catalog.revision_date = datetime.now(LOCALTZ) 626 catalog.fuzzy = False 627 628 with open(self.output_file, 'wb') as outfile: 629 write_po(outfile, catalog, width=self.width) 630 631 632class update_catalog(Command): 633 """Catalog merging command for use in ``setup.py`` scripts. 634 635 If correctly installed, this command is available to Setuptools-using 636 setup scripts automatically. For projects using plain old ``distutils``, 637 the command needs to be registered explicitly in ``setup.py``:: 638 639 from babel.messages.frontend import update_catalog 640 641 setup( 642 ... 643 cmdclass = {'update_catalog': update_catalog} 644 ) 645 646 .. versionadded:: 0.9 647 """ 648 649 description = 'update message catalogs from a POT file' 650 user_options = [ 651 ('domain=', 'D', 652 "domain of PO file (default 'messages')"), 653 ('input-file=', 'i', 654 'name of the input file'), 655 ('output-dir=', 'd', 656 'path to base directory containing the catalogs'), 657 ('output-file=', 'o', 658 "name of the output file (default " 659 "'<output_dir>/<locale>/LC_MESSAGES/<domain>.po')"), 660 ('omit-header', None, 661 "do not include msgid "" entry in header"), 662 ('locale=', 'l', 663 'locale of the catalog to compile'), 664 ('width=', 'w', 665 'set output line width (default 76)'), 666 ('no-wrap', None, 667 'do not break long message lines, longer than the output line width, ' 668 'into several lines'), 669 ('ignore-obsolete=', None, 670 'whether to omit obsolete messages from the output'), 671 ('no-fuzzy-matching', 'N', 672 'do not use fuzzy matching'), 673 ('update-header-comment', None, 674 'update target header comment'), 675 ('previous', None, 676 'keep previous msgids of translated messages'), 677 ] 678 boolean_options = [ 679 'omit-header', 'no-wrap', 'ignore-obsolete', 'no-fuzzy-matching', 680 'previous', 'update-header-comment', 681 ] 682 683 def initialize_options(self): 684 self.domain = 'messages' 685 self.input_file = None 686 self.output_dir = None 687 self.output_file = None 688 self.omit_header = False 689 self.locale = None 690 self.width = None 691 self.no_wrap = False 692 self.ignore_obsolete = False 693 self.no_fuzzy_matching = False 694 self.update_header_comment = False 695 self.previous = False 696 697 def finalize_options(self): 698 if not self.input_file: 699 raise DistutilsOptionError('you must specify the input file') 700 if not self.output_file and not self.output_dir: 701 raise DistutilsOptionError('you must specify the output file or ' 702 'directory') 703 if self.output_file and not self.locale: 704 raise DistutilsOptionError('you must specify the locale') 705 if self.no_wrap and self.width: 706 raise DistutilsOptionError("'--no-wrap' and '--width' are mutually " 707 "exclusive") 708 if not self.no_wrap and not self.width: 709 self.width = 76 710 elif self.width is not None: 711 self.width = int(self.width) 712 if self.no_fuzzy_matching and self.previous: 713 self.previous = False 714 715 def run(self): 716 po_files = [] 717 if not self.output_file: 718 if self.locale: 719 po_files.append((self.locale, 720 os.path.join(self.output_dir, self.locale, 721 'LC_MESSAGES', 722 self.domain + '.po'))) 723 else: 724 for locale in os.listdir(self.output_dir): 725 po_file = os.path.join(self.output_dir, locale, 726 'LC_MESSAGES', 727 self.domain + '.po') 728 if os.path.exists(po_file): 729 po_files.append((locale, po_file)) 730 else: 731 po_files.append((self.locale, self.output_file)) 732 733 if not po_files: 734 raise DistutilsOptionError('no message catalogs found') 735 736 domain = self.domain 737 if not domain: 738 domain = os.path.splitext(os.path.basename(self.input_file))[0] 739 740 with open(self.input_file, 'rb') as infile: 741 template = read_po(infile) 742 743 for locale, filename in po_files: 744 self.log.info('updating catalog %s based on %s', filename, self.input_file) 745 with open(filename, 'rb') as infile: 746 catalog = read_po(infile, locale=locale, domain=domain) 747 748 catalog.update( 749 template, self.no_fuzzy_matching, 750 update_header_comment=self.update_header_comment 751 ) 752 753 tmpname = os.path.join(os.path.dirname(filename), 754 tempfile.gettempprefix() + 755 os.path.basename(filename)) 756 try: 757 with open(tmpname, 'wb') as tmpfile: 758 write_po(tmpfile, catalog, 759 omit_header=self.omit_header, 760 ignore_obsolete=self.ignore_obsolete, 761 include_previous=self.previous, width=self.width) 762 except: 763 os.remove(tmpname) 764 raise 765 766 try: 767 os.rename(tmpname, filename) 768 except OSError: 769 # We're probably on Windows, which doesn't support atomic 770 # renames, at least not through Python 771 # If the error is in fact due to a permissions problem, that 772 # same error is going to be raised from one of the following 773 # operations 774 os.remove(filename) 775 shutil.copy(tmpname, filename) 776 os.remove(tmpname) 777 778 779class CommandLineInterface(object): 780 """Command-line interface. 781 782 This class provides a simple command-line interface to the message 783 extraction and PO file generation functionality. 784 """ 785 786 usage = '%%prog %s [options] %s' 787 version = '%%prog %s' % VERSION 788 commands = { 789 'compile': 'compile message catalogs to MO files', 790 'extract': 'extract messages from source files and generate a POT file', 791 'init': 'create new message catalogs from a POT file', 792 'update': 'update existing message catalogs from a POT file' 793 } 794 795 command_classes = { 796 'compile': compile_catalog, 797 'extract': extract_messages, 798 'init': init_catalog, 799 'update': update_catalog, 800 } 801 802 log = None # Replaced on instance level 803 804 def run(self, argv=None): 805 """Main entry point of the command-line interface. 806 807 :param argv: list of arguments passed on the command-line 808 """ 809 810 if argv is None: 811 argv = sys.argv 812 813 self.parser = optparse.OptionParser(usage=self.usage % ('command', '[args]'), 814 version=self.version) 815 self.parser.disable_interspersed_args() 816 self.parser.print_help = self._help 817 self.parser.add_option('--list-locales', dest='list_locales', 818 action='store_true', 819 help="print all known locales and exit") 820 self.parser.add_option('-v', '--verbose', action='store_const', 821 dest='loglevel', const=logging.DEBUG, 822 help='print as much as possible') 823 self.parser.add_option('-q', '--quiet', action='store_const', 824 dest='loglevel', const=logging.ERROR, 825 help='print as little as possible') 826 self.parser.set_defaults(list_locales=False, loglevel=logging.INFO) 827 828 options, args = self.parser.parse_args(argv[1:]) 829 830 self._configure_logging(options.loglevel) 831 if options.list_locales: 832 identifiers = localedata.locale_identifiers() 833 longest = max([len(identifier) for identifier in identifiers]) 834 identifiers.sort() 835 format = u'%%-%ds %%s' % (longest + 1) 836 for identifier in identifiers: 837 locale = Locale.parse(identifier) 838 output = format % (identifier, locale.english_name) 839 print(output.encode(sys.stdout.encoding or 840 getpreferredencoding() or 841 'ascii', 'replace')) 842 return 0 843 844 if not args: 845 self.parser.error('no valid command or option passed. ' 846 'Try the -h/--help option for more information.') 847 848 cmdname = args[0] 849 if cmdname not in self.commands: 850 self.parser.error('unknown command "%s"' % cmdname) 851 852 cmdinst = self._configure_command(cmdname, args[1:]) 853 return cmdinst.run() 854 855 def _configure_logging(self, loglevel): 856 self.log = logging.getLogger('babel') 857 self.log.setLevel(loglevel) 858 # Don't add a new handler for every instance initialization (#227), this 859 # would cause duplicated output when the CommandLineInterface as an 860 # normal Python class. 861 if self.log.handlers: 862 handler = self.log.handlers[0] 863 else: 864 handler = logging.StreamHandler() 865 self.log.addHandler(handler) 866 handler.setLevel(loglevel) 867 formatter = logging.Formatter('%(message)s') 868 handler.setFormatter(formatter) 869 870 def _help(self): 871 print(self.parser.format_help()) 872 print("commands:") 873 longest = max([len(command) for command in self.commands]) 874 format = " %%-%ds %%s" % max(8, longest + 1) 875 commands = sorted(self.commands.items()) 876 for name, description in commands: 877 print(format % (name, description)) 878 879 def _configure_command(self, cmdname, argv): 880 """ 881 :type cmdname: str 882 :type argv: list[str] 883 """ 884 cmdclass = self.command_classes[cmdname] 885 cmdinst = cmdclass() 886 if self.log: 887 cmdinst.log = self.log # Use our logger, not distutils'. 888 assert isinstance(cmdinst, Command) 889 cmdinst.initialize_options() 890 891 parser = optparse.OptionParser( 892 usage=self.usage % (cmdname, ''), 893 description=self.commands[cmdname] 894 ) 895 as_args = getattr(cmdclass, "as_args", ()) 896 for long, short, help in cmdclass.user_options: 897 name = long.strip("=") 898 default = getattr(cmdinst, name.replace('-', '_')) 899 strs = ["--%s" % name] 900 if short: 901 strs.append("-%s" % short) 902 strs.extend(cmdclass.option_aliases.get(name, ())) 903 choices = cmdclass.option_choices.get(name, None) 904 if name == as_args: 905 parser.usage += "<%s>" % name 906 elif name in cmdclass.boolean_options: 907 parser.add_option(*strs, action="store_true", help=help) 908 elif name in cmdclass.multiple_value_options: 909 parser.add_option(*strs, action="append", help=help, choices=choices) 910 else: 911 parser.add_option(*strs, help=help, default=default, choices=choices) 912 options, args = parser.parse_args(argv) 913 914 if as_args: 915 setattr(options, as_args.replace('-', '_'), args) 916 917 for key, value in vars(options).items(): 918 setattr(cmdinst, key, value) 919 920 try: 921 cmdinst.ensure_finalized() 922 except DistutilsOptionError as err: 923 parser.error(str(err)) 924 925 return cmdinst 926 927 928def main(): 929 return CommandLineInterface().run(sys.argv) 930 931 932def parse_mapping(fileobj, filename=None): 933 """Parse an extraction method mapping from a file-like object. 934 935 >>> buf = StringIO(''' 936 ... [extractors] 937 ... custom = mypackage.module:myfunc 938 ... 939 ... # Python source files 940 ... [python: **.py] 941 ... 942 ... # Genshi templates 943 ... [genshi: **/templates/**.html] 944 ... include_attrs = 945 ... [genshi: **/templates/**.txt] 946 ... template_class = genshi.template:TextTemplate 947 ... encoding = latin-1 948 ... 949 ... # Some custom extractor 950 ... [custom: **/custom/*.*] 951 ... ''') 952 953 >>> method_map, options_map = parse_mapping(buf) 954 >>> len(method_map) 955 4 956 957 >>> method_map[0] 958 ('**.py', 'python') 959 >>> options_map['**.py'] 960 {} 961 >>> method_map[1] 962 ('**/templates/**.html', 'genshi') 963 >>> options_map['**/templates/**.html']['include_attrs'] 964 '' 965 >>> method_map[2] 966 ('**/templates/**.txt', 'genshi') 967 >>> options_map['**/templates/**.txt']['template_class'] 968 'genshi.template:TextTemplate' 969 >>> options_map['**/templates/**.txt']['encoding'] 970 'latin-1' 971 972 >>> method_map[3] 973 ('**/custom/*.*', 'mypackage.module:myfunc') 974 >>> options_map['**/custom/*.*'] 975 {} 976 977 :param fileobj: a readable file-like object containing the configuration 978 text to parse 979 :see: `extract_from_directory` 980 """ 981 extractors = {} 982 method_map = [] 983 options_map = {} 984 985 parser = RawConfigParser() 986 parser._sections = OrderedDict(parser._sections) # We need ordered sections 987 988 if PY2: 989 parser.readfp(fileobj, filename) 990 else: 991 parser.read_file(fileobj, filename) 992 993 for section in parser.sections(): 994 if section == 'extractors': 995 extractors = dict(parser.items(section)) 996 else: 997 method, pattern = [part.strip() for part in section.split(':', 1)] 998 method_map.append((pattern, method)) 999 options_map[pattern] = dict(parser.items(section)) 1000 1001 if extractors: 1002 for idx, (pattern, method) in enumerate(method_map): 1003 if method in extractors: 1004 method = extractors[method] 1005 method_map[idx] = (pattern, method) 1006 1007 return method_map, options_map 1008 1009 1010def parse_keywords(strings=[]): 1011 """Parse keywords specifications from the given list of strings. 1012 1013 >>> kw = sorted(parse_keywords(['_', 'dgettext:2', 'dngettext:2,3', 'pgettext:1c,2']).items()) 1014 >>> for keyword, indices in kw: 1015 ... print((keyword, indices)) 1016 ('_', None) 1017 ('dgettext', (2,)) 1018 ('dngettext', (2, 3)) 1019 ('pgettext', ((1, 'c'), 2)) 1020 """ 1021 keywords = {} 1022 for string in strings: 1023 if ':' in string: 1024 funcname, indices = string.split(':') 1025 else: 1026 funcname, indices = string, None 1027 if funcname not in keywords: 1028 if indices: 1029 inds = [] 1030 for x in indices.split(','): 1031 if x[-1] == 'c': 1032 inds.append((int(x[:-1]), 'c')) 1033 else: 1034 inds.append(int(x)) 1035 indices = tuple(inds) 1036 keywords[funcname] = indices 1037 return keywords 1038 1039 1040if __name__ == '__main__': 1041 main() 1042