1# Copyright 2004-2021 Tom Rothamel <pytom@bishoujo.us> 2# 3# Permission is hereby granted, free of charge, to any person 4# obtaining a copy of this software and associated documentation files 5# (the "Software"), to deal in the Software without restriction, 6# including without limitation the rights to use, copy, modify, merge, 7# publish, distribute, sublicense, and/or sell copies of the Software, 8# and to permit persons to whom the Software is furnished to do so, 9# subject to the following conditions: 10# 11# The above copyright notice and this permission notice shall be 12# included in all copies or substantial portions of the Software. 13# 14# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 22from __future__ import division, absolute_import, with_statement, print_function, unicode_literals 23from renpy.compat import * 24 25import renpy 26renpy.update_path() 27 28import hashlib 29import re 30import collections 31import os 32import time 33import io 34import codecs 35 36################################################################################ 37# Script 38################################################################################ 39 40 41class ScriptTranslator(object): 42 43 def __init__(self): 44 45 # All languages we know about. 46 self.languages = set() 47 48 # A map from the translate identifier to the translate object used when the 49 # language is None. 50 self.default_translates = { } 51 52 # A map from (identifier, language) to the translate object used for that 53 # language. 54 self.language_translates = { } 55 56 # A list of (identifier, language) tuples that we need to chain together. 57 self.chain_worklist = [ ] 58 59 # A map from filename to a list of (label, translate) pairs found in 60 # that file. 61 self.file_translates = collections.defaultdict(list) 62 63 # A map from language to the StringTranslator for that language. 64 self.strings = collections.defaultdict(StringTranslator) 65 66 # A map from language to a list of TranslateBlock objects for 67 # that language. 68 self.block = collections.defaultdict(list) 69 70 # A map from language to a list of TranslateEarlyBlock objects for 71 # that language. 72 self.early_block = collections.defaultdict(list) 73 74 # A map from language to a list of TranslatePython objects for 75 # that language. 76 self.python = collections.defaultdict(list) 77 78 # A map from filename to a list of additional strings we've found 79 # in that file. 80 self.additional_strings = collections.defaultdict(list) 81 82 def count_translates(self): 83 """ 84 Return the number of dialogue blocks in the game. 85 """ 86 87 return len(self.default_translates) 88 89 def take_translates(self, nodes): 90 """ 91 Takes the translates out of the flattened list of statements, and stores 92 them into the dicts above. 93 """ 94 95 label = None 96 97 if not nodes: 98 return 99 100 TranslatePython = renpy.ast.TranslatePython 101 TranslateBlock = renpy.ast.TranslateBlock 102 TranslateEarlyBlock = renpy.ast.TranslateEarlyBlock 103 Menu = renpy.ast.Menu 104 UserStatement = renpy.ast.UserStatement 105 Translate = renpy.ast.Translate 106 107 filename = renpy.exports.unelide_filename(nodes[0].filename) 108 filename = os.path.normpath(os.path.abspath(filename)) 109 110 for n in nodes: 111 112 if not n.translation_relevant: 113 continue 114 115 if n.name.__class__ is not tuple: 116 if isinstance(n.name, basestring): 117 label = n.name 118 119 type_n = n.__class__ 120 121 if type_n is TranslatePython: 122 if n.language is not None: 123 self.languages.add(n.language) 124 self.python[n.language].append(n) 125 126 elif type_n is TranslateEarlyBlock: 127 if n.language is not None: 128 self.languages.add(n.language) 129 self.early_block[n.language].append(n) 130 131 elif type_n is TranslateBlock: 132 if n.language is not None: 133 self.languages.add(n.language) 134 self.block[n.language].append(n) 135 136 elif type_n is Menu: 137 138 for i in n.items: 139 s = i[0] 140 141 if s is None: 142 continue 143 144 self.additional_strings[filename].append((n.linenumber, s)) 145 146 elif type_n is UserStatement: 147 148 strings = n.call("translation_strings") 149 150 if strings is None: 151 continue 152 153 for s in strings: 154 self.additional_strings[filename].append((n.linenumber, s)) 155 156 elif type_n is Translate: 157 158 if n.language is None: 159 self.default_translates[n.identifier] = n 160 self.file_translates[filename].append((label, n)) 161 else: 162 self.languages.add(n.language) 163 self.language_translates[n.identifier, n.language] = n 164 self.chain_worklist.append((n.identifier, n.language)) 165 166 def chain_translates(self): 167 """ 168 Chains nodes in non-default translates together. 169 """ 170 171 unchained = [ ] 172 173 for identifier, language in self.chain_worklist: 174 175 if identifier not in self.default_translates: 176 unchained.append((identifier, language)) 177 continue 178 179 translate = self.language_translates[identifier, language] 180 next_node = self.default_translates[identifier].after 181 182 renpy.ast.chain_block(translate.block, next_node) 183 184 self.chain_worklist = unchained 185 186 def lookup_translate(self, identifier, alternate=None): 187 188 identifier = identifier.replace('.', '_') 189 language = renpy.game.preferences.language 190 191 if language is not None: 192 tl = self.language_translates.get((identifier, language), None) 193 194 if (tl is None) and alternate: 195 tl = self.language_translates.get((identifier, language), None) 196 197 else: 198 tl = None 199 200 if tl is None: 201 tl = self.default_translates[identifier] 202 203 return tl.block[0] 204 205 206def encode_say_string(s): 207 """ 208 Encodes a string in the format used by Ren'Py say statements. 209 """ 210 211 s = s.replace("\\", "\\\\") 212 s = s.replace("\n", "\\n") 213 s = s.replace("\"", "\\\"") 214 s = re.sub(r'(?<= ) ', '\\ ', s) 215 216 return "\"" + s + "\"" 217 218 219class Restructurer(object): 220 221 def __init__(self, children): 222 self.label = None 223 self.alternate = None 224 225 self.identifiers = set() 226 self.callback(children) 227 228 def id_exists(self, identifier): 229 if identifier in self.identifiers: 230 return True 231 232 if identifier in renpy.game.script.translator.default_translates: # @UndefinedVariable 233 return True 234 235 return False 236 237 def unique_identifier(self, label, digest): 238 239 if label is None: 240 base = digest 241 else: 242 base = label.replace(".", "_") + "_" + digest 243 244 i = 0 245 suffix = "" 246 247 while True: 248 249 identifier = base + suffix 250 251 if not self.id_exists(identifier): 252 break 253 254 i += 1 255 suffix = "_{0}".format(i) 256 257 return identifier 258 259 def create_translate(self, block): 260 """ 261 Creates an ast.Translate that wraps `block`. The block may only contain 262 translatable statements. 263 """ 264 265 md5 = hashlib.md5() 266 267 for i in block: 268 code = i.get_code() 269 md5.update((code + "\r\n").encode("utf-8")) 270 271 digest = md5.hexdigest()[:8] 272 273 identifier = self.unique_identifier(self.label, digest) 274 275 for i in block: 276 if isinstance(i, renpy.ast.Say): 277 identifier = getattr(i, "identifier", None) or identifier 278 279 self.identifiers.add(identifier) 280 281 if self.alternate is not None: 282 alternate = self.unique_identifier(self.alternate, digest) 283 self.identifiers.add(alternate) 284 else: 285 alternate = None 286 287 loc = (block[0].filename, block[0].linenumber) 288 289 tl = renpy.ast.Translate(loc, identifier, None, block, alternate=alternate) 290 tl.name = block[0].name + ("translate",) 291 292 ed = renpy.ast.EndTranslate(loc) 293 ed.name = block[0].name + ("end_translate",) 294 295 return [ tl, ed ] 296 297 def callback(self, children): 298 """ 299 This should be called with a list of statements. It restructures the statements 300 in the list so that translatable statements are contained within translation blocks. 301 """ 302 303 new_children = [ ] 304 group = [ ] 305 306 for i in children: 307 308 if isinstance(i, renpy.ast.Label): 309 if not i.hide: 310 311 if i.name.startswith("_"): 312 self.alternate = i.name 313 else: 314 self.label = i.name 315 self.alternate = None 316 317 if not isinstance(i, renpy.ast.Translate): 318 i.restructure(self.callback) 319 320 if isinstance(i, renpy.ast.Say): 321 group.append(i) 322 tl = self.create_translate(group) 323 new_children.extend(tl) 324 group = [ ] 325 326 elif i.translatable: 327 group.append(i) 328 329 else: 330 if group: 331 tl = self.create_translate(group) 332 new_children.extend(tl) 333 group = [ ] 334 335 new_children.append(i) 336 337 if group: 338 nodes = self.create_translate(group) 339 new_children.extend(nodes) 340 group = [ ] 341 342 children[:] = new_children 343 344 345def restructure(children): 346 Restructurer(children) 347 348################################################################################ 349# String Translation 350################################################################################ 351 352 353update_translations = ("RENPY_UPDATE_STRINGS" in os.environ) 354 355 356def quote_unicode(s): 357 s = s.replace("\\", "\\\\") 358 s = s.replace("\"", "\\\"") 359 s = s.replace("\a", "\\a") 360 s = s.replace("\b", "\\b") 361 s = s.replace("\f", "\\f") 362 s = s.replace("\n", "\\n") 363 s = s.replace("\r", "\\r") 364 s = s.replace("\t", "\\t") 365 s = s.replace("\v", "\\v") 366 367 return s 368 369 370class StringTranslator(object): 371 """ 372 This object stores the translations for a single language. It can also 373 buffer unknown translations, and write them to a file at game's end, if 374 we want that to happen. 375 """ 376 377 def __init__(self): 378 379 # A map from translation to translated string. 380 self.translations = { } 381 382 # A map from translation to the location of the translated string. 383 self.translation_loc = { } 384 385 # A list of unknown translations. 386 self.unknown = [ ] 387 388 def add(self, old, new, newloc): 389 if old in self.translations: 390 391 if old in self.translation_loc: 392 fn, line = self.translation_loc[old] 393 raise Exception("A translation for \"{}\" already exists at {}:{}.".format( 394 quote_unicode(old), fn, line)) 395 else: 396 raise Exception("A translation for \"{}\" already exists.".format( 397 quote_unicode(old))) 398 399 self.translations[old] = new 400 401 if newloc is not None: 402 self.translation_loc[old] = newloc 403 404 def translate(self, old): 405 406 new = self.translations.get(old, None) 407 408 if new is not None: 409 return new 410 411 if update_translations: 412 self.translations[old] = old 413 self.unknown.append(old) 414 415 # Remove {#...} tags. 416 if new is None: 417 notags = re.sub(r"\{\#.*?\}", "", old) 418 new = self.translations.get(notags, None) 419 420 if new is not None: 421 return new 422 423 return old 424 425 def write_updated_strings(self, language): 426 427 if not self.unknown: 428 return 429 430 if language is None: 431 fn = os.path.join(renpy.config.gamedir, "strings.rpy") 432 else: 433 fn = os.path.join(renpy.config.gamedir, renpy.config.tl_directory, language, "strings.rpy") 434 435 with renpy.translation.generation.open_tl_file(fn) as f: 436 f.write(u"translate {} strings:\n".format(language)) 437 f.write(u"\n") 438 439 for i in self.unknown: 440 441 i = quote_unicode(i) 442 443 f.write(u" old \"{}\"\n".format(i)) 444 f.write(u" new \"{}\"\n".format(i)) 445 f.write(u"\n") 446 447 448def add_string_translation(language, old, new, newloc): 449 450 tl = renpy.game.script.translator 451 stl = tl.strings[language] 452 tl.languages.add(language) 453 stl.add(old, new, newloc) 454 455 456Default = renpy.object.Sentinel("default") 457 458 459def translate_string(s, language=Default): 460 """ 461 :doc: translate_string 462 :name: renpy.translate_string 463 464 Returns `s` immediately translated into `language`. If `language` 465 is Default, uses the language set in the preferences. 466 Strings enclosed in this function will **not** be added 467 to the list of translatable strings. Note that the string may be 468 double-translated, if it matches a string translation when it 469 is displayed. 470 """ 471 472 if language is Default: 473 language = renpy.game.preferences.language 474 475 stl = renpy.game.script.translator.strings[language] # @UndefinedVariable 476 return stl.translate(s) 477 478 479def write_updated_strings(): 480 stl = renpy.game.script.translator.strings[renpy.game.preferences.language] # @UndefinedVariable 481 stl.write_updated_strings(renpy.game.preferences.language) 482 483################################################################################ 484# RPT Support 485# 486# RPT was the translation format used before 6.15. 487################################################################################ 488 489 490def load_rpt(fn): 491 """ 492 Loads the .rpt file `fn`. 493 """ 494 495 def unquote(s): 496 s = s.replace("\\n", "\n") 497 s = s.replace("\\\\", "\\") 498 return s 499 500 language = os.path.basename(fn).replace(".rpt", "") 501 502 old = None 503 504 with renpy.loader.load(fn) as f: 505 for l in f: 506 l = l.decode("utf-8") 507 l = l.rstrip() 508 509 if not l: 510 continue 511 512 if l[0] == '#': 513 continue 514 515 s = unquote(l[2:]) 516 517 if l[0] == '<': 518 if old: 519 raise Exception("{0} string {1!r} does not have a translation.".format(language, old)) 520 521 old = s 522 523 if l[0] == ">": 524 if old is None: 525 raise Exception("{0} translation {1!r} doesn't belong to a string.".format(language, s)) 526 527 add_string_translation(language, old, s, None) 528 old = None 529 530 if old is not None: 531 raise Exception("{0} string {1!r} does not have a translation.".format(language, old)) 532 533 534def load_all_rpts(): 535 """ 536 Loads all .rpt files. 537 """ 538 539 for fn in renpy.exports.list_files(): 540 if fn.endswith(".rpt"): 541 load_rpt(fn) 542 543################################################################################ 544# Changing language 545################################################################################ 546 547 548style_backup = None 549 550 551def init_translation(): 552 """ 553 Called before the game starts. 554 """ 555 556 global style_backup 557 style_backup = renpy.style.backup() # @UndefinedVariable 558 559 load_all_rpts() 560 561 renpy.store._init_language() # @UndefinedVariable 562 563 564old_language = "language never set" 565 566# A list of styles that have beend deferred to right before translate 567# styles are run. 568deferred_styles = [ ] 569 570 571def old_change_language(tl, language): 572 573 for i in deferred_styles: 574 i.apply() 575 576 def run_blocks(): 577 for i in tl.early_block[language]: 578 renpy.game.context().run(i.block[0]) 579 580 for i in tl.block[language]: 581 renpy.game.context().run(i.block[0]) 582 583 renpy.game.invoke_in_new_context(run_blocks) 584 585 for i in tl.python[language]: 586 renpy.python.py_exec_bytecode(i.code.bytecode) 587 588 for i in renpy.config.language_callbacks[language]: 589 i() 590 591 592def new_change_language(tl, language): 593 594 for i in tl.python[language]: 595 renpy.python.py_exec_bytecode(i.code.bytecode) 596 597 def run_blocks(): 598 for i in tl.early_block[language]: 599 renpy.game.context().run(i.block[0]) 600 601 renpy.game.invoke_in_new_context(run_blocks) 602 603 for i in renpy.config.language_callbacks[language]: 604 i() 605 606 for i in deferred_styles: 607 i.apply() 608 609 def run_blocks(): 610 for i in tl.block[language]: 611 renpy.game.context().run(i.block[0]) 612 613 renpy.game.invoke_in_new_context(run_blocks) 614 615 renpy.config.init_system_styles() 616 617 618def change_language(language, force=False): 619 """ 620 :doc: translation_functions 621 622 Changes the current language to `language`, which can be a string or 623 None to use the default language. 624 """ 625 626 global old_language 627 628 if old_language != language: 629 renpy.store._history_list = renpy.store.list() 630 renpy.store.nvl_list = renpy.store.list() 631 632 renpy.game.preferences.language = language 633 if old_language == language and not force: 634 return 635 636 tl = renpy.game.script.translator 637 638 renpy.style.restore(style_backup) # @UndefinedVariable 639 renpy.style.rebuild() # @UndefinedVariable 640 641 for i in renpy.config.translate_clean_stores: 642 renpy.python.clean_store(i) 643 644 if renpy.config.new_translate_order: 645 new_change_language(tl, language) 646 else: 647 old_change_language(tl, language) 648 649 for i in renpy.config.change_language_callbacks: 650 i() 651 652 # Reset various parts of the system. Most notably, this clears the image 653 # cache, letting us load translated images. 654 renpy.exports.free_memory() 655 656 # Rebuild the styles. 657 renpy.style.rebuild() # @UndefinedVariable 658 659 for i in renpy.config.translate_clean_stores: 660 renpy.python.reset_store_changes(i) 661 662 # Restart the interaction. 663 renpy.exports.restart_interaction() 664 665 if language != old_language: 666 renpy.exports.block_rollback() 667 668 old_language = language 669 670 671def check_language(): 672 """ 673 Checks to see if the language has changed. If it has, jump to the start 674 of the current translation block. 675 """ 676 677 ctx = renpy.game.contexts[-1] 678 preferences = renpy.game.preferences 679 680 # Deal with a changed language. 681 if ctx.translate_language != preferences.language: 682 ctx.translate_language = preferences.language 683 684 tid = ctx.translate_identifier 685 686 if tid is not None: 687 node = renpy.game.script.translator.lookup_translate(tid) # @UndefinedVariable 688 689 if node is not None: 690 raise renpy.game.JumpException(node.name) 691 692 693def known_languages(): 694 """ 695 :doc: translation_functions 696 697 Returns the set of known languages. This does not include the default 698 language, None. 699 """ 700 701 return { i for i in renpy.game.script.translator.languages if i is not None } # @UndefinedVariable 702 703################################################################################ 704# Detect language 705################################################################################ 706 707 708locales = { 709 "ab": "abkhazian", 710 "aa": "afar", 711 "af": "afrikaans", 712 "ak": "akan", 713 "sq": "albanian", 714 "am": "amharic", 715 "ar": "arabic", 716 "an": "aragonese", 717 "hy": "armenian", 718 "as": "assamese", 719 "av": "avaric", 720 "ae": "avestan", 721 "ay": "aymara", 722 "az": "azerbaijani", 723 "bm": "bambara", 724 "ba": "bashkir", 725 "eu": "basque", 726 "be": "belarusian", 727 "bn": "bengali", 728 "bh": "bihari", 729 "bi": "bislama", 730 "bs": "bosnian", 731 "br": "breton", 732 "bg": "bulgarian", 733 "my": "burmese", 734 "ca": "catalan", 735 "ch": "chamorro", 736 "ce": "chechen", 737 "ny": "chewa", 738 "cv": "chuvash", 739 "kw": "cornish", 740 "co": "corsican", 741 "cr": "cree", 742 "hr": "croatian", 743 "cs": "czech", 744 "da": "danish", 745 "dv": "maldivian", 746 "nl": "dutch", 747 "dz": "dzongkha", 748 "en": "english", 749 "et": "estonian", 750 "ee": "ewe", 751 "fo": "faroese", 752 "fj": "fijian", 753 "fi": "finnish", 754 "fr": "french", 755 "ff": "fulah", 756 "gl": "galician", 757 "ka": "georgian", 758 "de": "german", 759 "el": "greek", 760 "gn": "guaran", 761 "gu": "gujarati", 762 "ht": "haitian", 763 "ha": "hausa", 764 "he": "hebrew", 765 "hz": "herero", 766 "hi": "hindi", 767 "ho": "hiri_motu", 768 "hu": "hungarian", 769 "id": "indonesian", 770 "ga": "irish", 771 "ig": "igbo", 772 "ik": "inupiaq", 773 "is": "icelandic", 774 "it": "italian", 775 "iu": "inuktitut", 776 "ja": "japanese", 777 "jv": "javanese", 778 "kl": "greenlandic", 779 "kn": "kannada", 780 "kr": "kanuri", 781 "ks": "kashmiri", 782 "kk": "kazakh", 783 "km": "khmer", 784 "ki": "kikuyu", 785 "rw": "kinyarwanda", 786 "ky": "kirghiz", 787 "kv": "komi", 788 "kg": "kongo", 789 "ko": "korean", 790 "ku": "kurdish", 791 "kj": "kuanyama", 792 "la": "latin", 793 "lb": "luxembourgish", 794 "lg": "ganda", 795 "li": "limburgan", 796 "ln": "lingala", 797 "lo": "lao", 798 "lt": "lithuanian", 799 "lv": "latvian", 800 "gv": "manx", 801 "mk": "macedonian", 802 "mg": "malagasy", 803 "ms": "malay", 804 "ml": "malayalam", 805 "mt": "maltese", 806 "mi": "maori", 807 "mr": "marathi", 808 "mh": "marshallese", 809 "mn": "mongolian", 810 "na": "nauru", 811 "nv": "navaho", 812 "ne": "nepali", 813 "ng": "ndonga", 814 "no": "norwegian", 815 "ii": "nuosu", 816 "nr": "ndebele", 817 "oc": "occitan", 818 "oj": "ojibwa", 819 "om": "oromo", 820 "or": "oriya", 821 "os": "ossetian", 822 "pa": "panjabi", 823 "pi": "pali", 824 "fa": "persian", 825 "pl": "polish", 826 "ps": "pashto", 827 "pt": "portuguese", 828 "qu": "quechua", 829 "rm": "romansh", 830 "rn": "rundi", 831 "ro": "romanian", 832 "ru": "russian", 833 "sa": "sanskrit", 834 "sc": "sardinian", 835 "sd": "sindhi", 836 "se": "sami", 837 "sm": "samoan", 838 "sg": "sango", 839 "sr": "serbian", 840 "gd": "gaelic", 841 "sn": "shona", 842 "si": "sinhala", 843 "sk": "slovak", 844 "sl": "slovene", 845 "so": "somali", 846 "st": "sotho", 847 "es": "spanish", 848 "su": "sundanese", 849 "sw": "swahili", 850 "ss": "swati", 851 "sv": "swedish", 852 "ta": "tamil", 853 "te": "telugu", 854 "tg": "tajik", 855 "th": "thai", 856 "ti": "tigrinya", 857 "bo": "tibetan", 858 "tk": "turkmen", 859 "tl": "tagalog", 860 "tn": "tswana", 861 "to": "tongan", 862 "tr": "turkish", 863 "ts": "tsonga", 864 "tt": "tatar", 865 "tw": "twi", 866 "ty": "tahitian", 867 "ug": "uighur", 868 "uk": "ukrainian", 869 "ur": "urdu", 870 "uz": "uzbek", 871 "ve": "venda", 872 "vi": "vietnamese", 873 "wa": "walloon", 874 "cy": "welsh", 875 "wo": "wolof", 876 "fy": "frisian", 877 "xh": "xhosa", 878 "yi": "yiddish", 879 "yo": "yoruba", 880 "za": "zhuang", 881 "zu": "zulu", 882 "chs": "simplified_chinese", 883 "cht": "traditional_chinese", 884 "zh": "traditional_chinese", 885} 886 887 888def detect_user_locale(): 889 import locale 890 if renpy.windows: 891 import ctypes 892 windll = ctypes.windll.kernel32 893 locale_name = locale.windows_locale.get(windll.GetUserDefaultUILanguage()) 894 elif renpy.android: 895 from jnius import autoclass 896 Locale = autoclass('java.util.Locale') 897 locale_name = str(Locale.getDefault().getLanguage()) 898 elif renpy.ios: 899 import pyobjus 900 NSLocale = pyobjus.autoclass("NSLocale") 901 languages = NSLocale.preferredLanguages() 902 locale_name = languages.objectAtIndex_(0).UTF8String().decode("utf-8") 903 locale_name.replace("-", "_") 904 else: 905 locale_name = locale.getdefaultlocale() 906 if locale_name is not None: 907 locale_name = locale_name[0] 908 909 if locale_name is None: 910 return None, None 911 912 normalize = locale.normalize(locale_name) 913 if normalize == locale_name: 914 language = region = locale_name 915 else: 916 locale_name = normalize 917 if '.' in locale_name: 918 locale_name, _ = locale_name.split('.', 1) 919 language, region = locale_name.lower().split("_") 920 return language, region 921