1# Copyright 2004-2021 Tom Rothamel <pytom@bishoujo.us>
2#
3# Permission is hereby granted, free of charge, to any person
4# obtaining a copy of this software and associated documentation files
5# (the "Software"), to deal in the Software without restriction,
6# including without limitation the rights to use, copy, modify, merge,
7# publish, distribute, sublicense, and/or sell copies of the Software,
8# and to permit persons to whom the Software is furnished to do so,
9# subject to the following conditions:
10#
11# The above copyright notice and this permission notice shall be
12# included in all copies or substantial portions of the Software.
13#
14# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
18# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
19# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
20# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
21
22from __future__ import division, absolute_import, with_statement, print_function, unicode_literals
23from renpy.compat import *
24
25import renpy
26renpy.update_path()
27
28import hashlib
29import re
30import collections
31import os
32import time
33import io
34import codecs
35
36################################################################################
37# Script
38################################################################################
39
40
41class ScriptTranslator(object):
42
43    def __init__(self):
44
45        # All languages we know about.
46        self.languages = set()
47
48        # A map from the translate identifier to the translate object used when the
49        # language is None.
50        self.default_translates = { }
51
52        # A map from (identifier, language) to the translate object used for that
53        # language.
54        self.language_translates = { }
55
56        # A list of (identifier, language) tuples that we need to chain together.
57        self.chain_worklist = [ ]
58
59        # A map from filename to a list of (label, translate) pairs found in
60        # that file.
61        self.file_translates = collections.defaultdict(list)
62
63        # A map from language to the StringTranslator for that language.
64        self.strings = collections.defaultdict(StringTranslator)
65
66        # A map from language to a list of TranslateBlock objects for
67        # that language.
68        self.block = collections.defaultdict(list)
69
70        # A map from language to a list of TranslateEarlyBlock objects for
71        # that language.
72        self.early_block = collections.defaultdict(list)
73
74        # A map from language to a list of TranslatePython objects for
75        # that language.
76        self.python = collections.defaultdict(list)
77
78        # A map from filename to a list of additional strings we've found
79        # in that file.
80        self.additional_strings = collections.defaultdict(list)
81
82    def count_translates(self):
83        """
84        Return the number of dialogue blocks in the game.
85        """
86
87        return len(self.default_translates)
88
89    def take_translates(self, nodes):
90        """
91        Takes the translates out of the flattened list of statements, and stores
92        them into the dicts above.
93        """
94
95        label = None
96
97        if not nodes:
98            return
99
100        TranslatePython = renpy.ast.TranslatePython
101        TranslateBlock = renpy.ast.TranslateBlock
102        TranslateEarlyBlock = renpy.ast.TranslateEarlyBlock
103        Menu = renpy.ast.Menu
104        UserStatement = renpy.ast.UserStatement
105        Translate = renpy.ast.Translate
106
107        filename = renpy.exports.unelide_filename(nodes[0].filename)
108        filename = os.path.normpath(os.path.abspath(filename))
109
110        for n in nodes:
111
112            if not n.translation_relevant:
113                continue
114
115            if n.name.__class__ is not tuple:
116                if isinstance(n.name, basestring):
117                    label = n.name
118
119            type_n = n.__class__
120
121            if type_n is TranslatePython:
122                if n.language is not None:
123                    self.languages.add(n.language)
124                self.python[n.language].append(n)
125
126            elif type_n is TranslateEarlyBlock:
127                if n.language is not None:
128                    self.languages.add(n.language)
129                self.early_block[n.language].append(n)
130
131            elif type_n is TranslateBlock:
132                if n.language is not None:
133                    self.languages.add(n.language)
134                self.block[n.language].append(n)
135
136            elif type_n is Menu:
137
138                for i in n.items:
139                    s = i[0]
140
141                    if s is None:
142                        continue
143
144                    self.additional_strings[filename].append((n.linenumber, s))
145
146            elif type_n is UserStatement:
147
148                strings = n.call("translation_strings")
149
150                if strings is None:
151                    continue
152
153                for s in strings:
154                    self.additional_strings[filename].append((n.linenumber, s))
155
156            elif type_n is Translate:
157
158                if n.language is None:
159                    self.default_translates[n.identifier] = n
160                    self.file_translates[filename].append((label, n))
161                else:
162                    self.languages.add(n.language)
163                    self.language_translates[n.identifier, n.language] = n
164                    self.chain_worklist.append((n.identifier, n.language))
165
166    def chain_translates(self):
167        """
168        Chains nodes in non-default translates together.
169        """
170
171        unchained = [ ]
172
173        for identifier, language in self.chain_worklist:
174
175            if identifier not in self.default_translates:
176                unchained.append((identifier, language))
177                continue
178
179            translate = self.language_translates[identifier, language]
180            next_node = self.default_translates[identifier].after
181
182            renpy.ast.chain_block(translate.block, next_node)
183
184        self.chain_worklist = unchained
185
186    def lookup_translate(self, identifier, alternate=None):
187
188        identifier = identifier.replace('.', '_')
189        language = renpy.game.preferences.language
190
191        if language is not None:
192            tl = self.language_translates.get((identifier, language), None)
193
194            if (tl is None) and alternate:
195                tl = self.language_translates.get((identifier, language), None)
196
197        else:
198            tl = None
199
200        if tl is None:
201            tl = self.default_translates[identifier]
202
203        return tl.block[0]
204
205
206def encode_say_string(s):
207    """
208    Encodes a string in the format used by Ren'Py say statements.
209    """
210
211    s = s.replace("\\", "\\\\")
212    s = s.replace("\n", "\\n")
213    s = s.replace("\"", "\\\"")
214    s = re.sub(r'(?<= ) ', '\\ ', s)
215
216    return "\"" + s + "\""
217
218
219class Restructurer(object):
220
221    def __init__(self, children):
222        self.label = None
223        self.alternate = None
224
225        self.identifiers = set()
226        self.callback(children)
227
228    def id_exists(self, identifier):
229        if identifier in self.identifiers:
230            return True
231
232        if identifier in renpy.game.script.translator.default_translates: # @UndefinedVariable
233            return True
234
235        return False
236
237    def unique_identifier(self, label, digest):
238
239        if label is None:
240            base = digest
241        else:
242            base = label.replace(".", "_") + "_" + digest
243
244        i = 0
245        suffix = ""
246
247        while True:
248
249            identifier = base + suffix
250
251            if not self.id_exists(identifier):
252                break
253
254            i += 1
255            suffix = "_{0}".format(i)
256
257        return identifier
258
259    def create_translate(self, block):
260        """
261        Creates an ast.Translate that wraps `block`. The block may only contain
262        translatable statements.
263        """
264
265        md5 = hashlib.md5()
266
267        for i in block:
268            code = i.get_code()
269            md5.update((code + "\r\n").encode("utf-8"))
270
271        digest = md5.hexdigest()[:8]
272
273        identifier = self.unique_identifier(self.label, digest)
274
275        for i in block:
276            if isinstance(i, renpy.ast.Say):
277                identifier = getattr(i, "identifier", None) or identifier
278
279        self.identifiers.add(identifier)
280
281        if self.alternate is not None:
282            alternate = self.unique_identifier(self.alternate, digest)
283            self.identifiers.add(alternate)
284        else:
285            alternate = None
286
287        loc = (block[0].filename, block[0].linenumber)
288
289        tl = renpy.ast.Translate(loc, identifier, None, block, alternate=alternate)
290        tl.name = block[0].name + ("translate",)
291
292        ed = renpy.ast.EndTranslate(loc)
293        ed.name = block[0].name + ("end_translate",)
294
295        return [ tl, ed ]
296
297    def callback(self, children):
298        """
299        This should be called with a list of statements. It restructures the statements
300        in the list so that translatable statements are contained within translation blocks.
301        """
302
303        new_children = [ ]
304        group = [ ]
305
306        for i in children:
307
308            if isinstance(i, renpy.ast.Label):
309                if not i.hide:
310
311                    if i.name.startswith("_"):
312                        self.alternate = i.name
313                    else:
314                        self.label = i.name
315                        self.alternate = None
316
317            if not isinstance(i, renpy.ast.Translate):
318                i.restructure(self.callback)
319
320            if isinstance(i, renpy.ast.Say):
321                group.append(i)
322                tl = self.create_translate(group)
323                new_children.extend(tl)
324                group = [ ]
325
326            elif i.translatable:
327                group.append(i)
328
329            else:
330                if group:
331                    tl = self.create_translate(group)
332                    new_children.extend(tl)
333                    group = [ ]
334
335                new_children.append(i)
336
337        if group:
338            nodes = self.create_translate(group)
339            new_children.extend(nodes)
340            group = [ ]
341
342        children[:] = new_children
343
344
345def restructure(children):
346    Restructurer(children)
347
348################################################################################
349# String Translation
350################################################################################
351
352
353update_translations = ("RENPY_UPDATE_STRINGS" in os.environ)
354
355
356def quote_unicode(s):
357    s = s.replace("\\", "\\\\")
358    s = s.replace("\"", "\\\"")
359    s = s.replace("\a", "\\a")
360    s = s.replace("\b", "\\b")
361    s = s.replace("\f", "\\f")
362    s = s.replace("\n", "\\n")
363    s = s.replace("\r", "\\r")
364    s = s.replace("\t", "\\t")
365    s = s.replace("\v", "\\v")
366
367    return s
368
369
370class StringTranslator(object):
371    """
372    This object stores the translations for a single language. It can also
373    buffer unknown translations, and write them to a file at game's end, if
374    we want that to happen.
375    """
376
377    def __init__(self):
378
379        # A map from translation to translated string.
380        self.translations = { }
381
382        # A map from translation to the location of the translated string.
383        self.translation_loc = { }
384
385        # A list of unknown translations.
386        self.unknown = [ ]
387
388    def add(self, old, new, newloc):
389        if old in self.translations:
390
391            if old in self.translation_loc:
392                fn, line = self.translation_loc[old]
393                raise Exception("A translation for \"{}\" already exists at {}:{}.".format(
394                    quote_unicode(old), fn, line))
395            else:
396                raise Exception("A translation for \"{}\" already exists.".format(
397                    quote_unicode(old)))
398
399        self.translations[old] = new
400
401        if newloc is not None:
402            self.translation_loc[old] = newloc
403
404    def translate(self, old):
405
406        new = self.translations.get(old, None)
407
408        if new is not None:
409            return new
410
411        if update_translations:
412            self.translations[old] = old
413            self.unknown.append(old)
414
415        # Remove {#...} tags.
416        if new is None:
417            notags = re.sub(r"\{\#.*?\}", "", old)
418            new = self.translations.get(notags, None)
419
420        if new is not None:
421            return new
422
423        return old
424
425    def write_updated_strings(self, language):
426
427        if not self.unknown:
428            return
429
430        if language is None:
431            fn = os.path.join(renpy.config.gamedir, "strings.rpy")
432        else:
433            fn = os.path.join(renpy.config.gamedir, renpy.config.tl_directory, language, "strings.rpy")
434
435        with renpy.translation.generation.open_tl_file(fn) as f:
436            f.write(u"translate {} strings:\n".format(language))
437            f.write(u"\n")
438
439            for i in self.unknown:
440
441                i = quote_unicode(i)
442
443                f.write(u"    old \"{}\"\n".format(i))
444                f.write(u"    new \"{}\"\n".format(i))
445                f.write(u"\n")
446
447
448def add_string_translation(language, old, new, newloc):
449
450    tl = renpy.game.script.translator
451    stl = tl.strings[language]
452    tl.languages.add(language)
453    stl.add(old, new, newloc)
454
455
456Default = renpy.object.Sentinel("default")
457
458
459def translate_string(s, language=Default):
460    """
461    :doc: translate_string
462    :name: renpy.translate_string
463
464    Returns `s` immediately translated into `language`. If `language`
465    is Default, uses the language set in the preferences.
466    Strings enclosed in this function will **not** be added
467    to the list of translatable strings. Note that the string may be
468    double-translated, if it matches a string translation when it
469    is displayed.
470    """
471
472    if language is Default:
473        language = renpy.game.preferences.language
474
475    stl = renpy.game.script.translator.strings[language] # @UndefinedVariable
476    return stl.translate(s)
477
478
479def write_updated_strings():
480    stl = renpy.game.script.translator.strings[renpy.game.preferences.language] # @UndefinedVariable
481    stl.write_updated_strings(renpy.game.preferences.language)
482
483################################################################################
484# RPT Support
485#
486# RPT was the translation format used before 6.15.
487################################################################################
488
489
490def load_rpt(fn):
491    """
492    Loads the .rpt file `fn`.
493    """
494
495    def unquote(s):
496        s = s.replace("\\n", "\n")
497        s = s.replace("\\\\", "\\")
498        return s
499
500    language = os.path.basename(fn).replace(".rpt", "")
501
502    old = None
503
504    with renpy.loader.load(fn) as f:
505        for l in f:
506            l = l.decode("utf-8")
507            l = l.rstrip()
508
509            if not l:
510                continue
511
512            if l[0] == '#':
513                continue
514
515            s = unquote(l[2:])
516
517            if l[0] == '<':
518                if old:
519                    raise Exception("{0} string {1!r} does not have a translation.".format(language, old))
520
521                old = s
522
523            if l[0] == ">":
524                if old is None:
525                    raise Exception("{0} translation {1!r} doesn't belong to a string.".format(language, s))
526
527                add_string_translation(language, old, s, None)
528                old = None
529
530    if old is not None:
531        raise Exception("{0} string {1!r} does not have a translation.".format(language, old))
532
533
534def load_all_rpts():
535    """
536    Loads all .rpt files.
537    """
538
539    for fn in renpy.exports.list_files():
540        if fn.endswith(".rpt"):
541            load_rpt(fn)
542
543################################################################################
544# Changing language
545################################################################################
546
547
548style_backup = None
549
550
551def init_translation():
552    """
553    Called before the game starts.
554    """
555
556    global style_backup
557    style_backup = renpy.style.backup() # @UndefinedVariable
558
559    load_all_rpts()
560
561    renpy.store._init_language() # @UndefinedVariable
562
563
564old_language = "language never set"
565
566# A list of styles that have beend deferred to right before translate
567# styles are run.
568deferred_styles = [ ]
569
570
571def old_change_language(tl, language):
572
573    for i in deferred_styles:
574        i.apply()
575
576    def run_blocks():
577        for i in tl.early_block[language]:
578            renpy.game.context().run(i.block[0])
579
580        for i in tl.block[language]:
581            renpy.game.context().run(i.block[0])
582
583    renpy.game.invoke_in_new_context(run_blocks)
584
585    for i in tl.python[language]:
586        renpy.python.py_exec_bytecode(i.code.bytecode)
587
588    for i in renpy.config.language_callbacks[language]:
589        i()
590
591
592def new_change_language(tl, language):
593
594    for i in tl.python[language]:
595        renpy.python.py_exec_bytecode(i.code.bytecode)
596
597    def run_blocks():
598        for i in tl.early_block[language]:
599            renpy.game.context().run(i.block[0])
600
601    renpy.game.invoke_in_new_context(run_blocks)
602
603    for i in renpy.config.language_callbacks[language]:
604        i()
605
606    for i in deferred_styles:
607        i.apply()
608
609    def run_blocks():
610        for i in tl.block[language]:
611            renpy.game.context().run(i.block[0])
612
613    renpy.game.invoke_in_new_context(run_blocks)
614
615    renpy.config.init_system_styles()
616
617
618def change_language(language, force=False):
619    """
620    :doc: translation_functions
621
622    Changes the current language to `language`, which can be a string or
623    None to use the default language.
624    """
625
626    global old_language
627
628    if old_language != language:
629        renpy.store._history_list = renpy.store.list()
630        renpy.store.nvl_list = renpy.store.list()
631
632    renpy.game.preferences.language = language
633    if old_language == language and not force:
634        return
635
636    tl = renpy.game.script.translator
637
638    renpy.style.restore(style_backup) # @UndefinedVariable
639    renpy.style.rebuild() # @UndefinedVariable
640
641    for i in renpy.config.translate_clean_stores:
642        renpy.python.clean_store(i)
643
644    if renpy.config.new_translate_order:
645        new_change_language(tl, language)
646    else:
647        old_change_language(tl, language)
648
649    for i in renpy.config.change_language_callbacks:
650        i()
651
652    # Reset various parts of the system. Most notably, this clears the image
653    # cache, letting us load translated images.
654    renpy.exports.free_memory()
655
656    # Rebuild the styles.
657    renpy.style.rebuild() # @UndefinedVariable
658
659    for i in renpy.config.translate_clean_stores:
660        renpy.python.reset_store_changes(i)
661
662    # Restart the interaction.
663    renpy.exports.restart_interaction()
664
665    if language != old_language:
666        renpy.exports.block_rollback()
667
668        old_language = language
669
670
671def check_language():
672    """
673    Checks to see if the language has changed. If it has, jump to the start
674    of the current translation block.
675    """
676
677    ctx = renpy.game.contexts[-1]
678    preferences = renpy.game.preferences
679
680    # Deal with a changed language.
681    if ctx.translate_language != preferences.language:
682        ctx.translate_language = preferences.language
683
684        tid = ctx.translate_identifier
685
686        if tid is not None:
687            node = renpy.game.script.translator.lookup_translate(tid) # @UndefinedVariable
688
689            if node is not None:
690                raise renpy.game.JumpException(node.name)
691
692
693def known_languages():
694    """
695    :doc: translation_functions
696
697    Returns the set of known languages. This does not include the default
698    language, None.
699    """
700
701    return { i for i in renpy.game.script.translator.languages if i is not None } # @UndefinedVariable
702
703################################################################################
704# Detect language
705################################################################################
706
707
708locales = {
709    "ab": "abkhazian",
710    "aa": "afar",
711    "af": "afrikaans",
712    "ak": "akan",
713    "sq": "albanian",
714    "am": "amharic",
715    "ar": "arabic",
716    "an": "aragonese",
717    "hy": "armenian",
718    "as": "assamese",
719    "av": "avaric",
720    "ae": "avestan",
721    "ay": "aymara",
722    "az": "azerbaijani",
723    "bm": "bambara",
724    "ba": "bashkir",
725    "eu": "basque",
726    "be": "belarusian",
727    "bn": "bengali",
728    "bh": "bihari",
729    "bi": "bislama",
730    "bs": "bosnian",
731    "br": "breton",
732    "bg": "bulgarian",
733    "my": "burmese",
734    "ca": "catalan",
735    "ch": "chamorro",
736    "ce": "chechen",
737    "ny": "chewa",
738    "cv": "chuvash",
739    "kw": "cornish",
740    "co": "corsican",
741    "cr": "cree",
742    "hr": "croatian",
743    "cs": "czech",
744    "da": "danish",
745    "dv": "maldivian",
746    "nl": "dutch",
747    "dz": "dzongkha",
748    "en": "english",
749    "et": "estonian",
750    "ee": "ewe",
751    "fo": "faroese",
752    "fj": "fijian",
753    "fi": "finnish",
754    "fr": "french",
755    "ff": "fulah",
756    "gl": "galician",
757    "ka": "georgian",
758    "de": "german",
759    "el": "greek",
760    "gn": "guaran",
761    "gu": "gujarati",
762    "ht": "haitian",
763    "ha": "hausa",
764    "he": "hebrew",
765    "hz": "herero",
766    "hi": "hindi",
767    "ho": "hiri_motu",
768    "hu": "hungarian",
769    "id": "indonesian",
770    "ga": "irish",
771    "ig": "igbo",
772    "ik": "inupiaq",
773    "is": "icelandic",
774    "it": "italian",
775    "iu": "inuktitut",
776    "ja": "japanese",
777    "jv": "javanese",
778    "kl": "greenlandic",
779    "kn": "kannada",
780    "kr": "kanuri",
781    "ks": "kashmiri",
782    "kk": "kazakh",
783    "km": "khmer",
784    "ki": "kikuyu",
785    "rw": "kinyarwanda",
786    "ky": "kirghiz",
787    "kv": "komi",
788    "kg": "kongo",
789    "ko": "korean",
790    "ku": "kurdish",
791    "kj": "kuanyama",
792    "la": "latin",
793    "lb": "luxembourgish",
794    "lg": "ganda",
795    "li": "limburgan",
796    "ln": "lingala",
797    "lo": "lao",
798    "lt": "lithuanian",
799    "lv": "latvian",
800    "gv": "manx",
801    "mk": "macedonian",
802    "mg": "malagasy",
803    "ms": "malay",
804    "ml": "malayalam",
805    "mt": "maltese",
806    "mi": "maori",
807    "mr": "marathi",
808    "mh": "marshallese",
809    "mn": "mongolian",
810    "na": "nauru",
811    "nv": "navaho",
812    "ne": "nepali",
813    "ng": "ndonga",
814    "no": "norwegian",
815    "ii": "nuosu",
816    "nr": "ndebele",
817    "oc": "occitan",
818    "oj": "ojibwa",
819    "om": "oromo",
820    "or": "oriya",
821    "os": "ossetian",
822    "pa": "panjabi",
823    "pi": "pali",
824    "fa": "persian",
825    "pl": "polish",
826    "ps": "pashto",
827    "pt": "portuguese",
828    "qu": "quechua",
829    "rm": "romansh",
830    "rn": "rundi",
831    "ro": "romanian",
832    "ru": "russian",
833    "sa": "sanskrit",
834    "sc": "sardinian",
835    "sd": "sindhi",
836    "se": "sami",
837    "sm": "samoan",
838    "sg": "sango",
839    "sr": "serbian",
840    "gd": "gaelic",
841    "sn": "shona",
842    "si": "sinhala",
843    "sk": "slovak",
844    "sl": "slovene",
845    "so": "somali",
846    "st": "sotho",
847    "es": "spanish",
848    "su": "sundanese",
849    "sw": "swahili",
850    "ss": "swati",
851    "sv": "swedish",
852    "ta": "tamil",
853    "te": "telugu",
854    "tg": "tajik",
855    "th": "thai",
856    "ti": "tigrinya",
857    "bo": "tibetan",
858    "tk": "turkmen",
859    "tl": "tagalog",
860    "tn": "tswana",
861    "to": "tongan",
862    "tr": "turkish",
863    "ts": "tsonga",
864    "tt": "tatar",
865    "tw": "twi",
866    "ty": "tahitian",
867    "ug": "uighur",
868    "uk": "ukrainian",
869    "ur": "urdu",
870    "uz": "uzbek",
871    "ve": "venda",
872    "vi": "vietnamese",
873    "wa": "walloon",
874    "cy": "welsh",
875    "wo": "wolof",
876    "fy": "frisian",
877    "xh": "xhosa",
878    "yi": "yiddish",
879    "yo": "yoruba",
880    "za": "zhuang",
881    "zu": "zulu",
882    "chs": "simplified_chinese",
883    "cht": "traditional_chinese",
884    "zh": "traditional_chinese",
885}
886
887
888def detect_user_locale():
889    import locale
890    if renpy.windows:
891        import ctypes
892        windll = ctypes.windll.kernel32
893        locale_name = locale.windows_locale.get(windll.GetUserDefaultUILanguage())
894    elif renpy.android:
895        from jnius import autoclass
896        Locale = autoclass('java.util.Locale')
897        locale_name = str(Locale.getDefault().getLanguage())
898    elif renpy.ios:
899        import pyobjus
900        NSLocale = pyobjus.autoclass("NSLocale")
901        languages = NSLocale.preferredLanguages()
902        locale_name = languages.objectAtIndex_(0).UTF8String().decode("utf-8")
903        locale_name.replace("-", "_")
904    else:
905        locale_name = locale.getdefaultlocale()
906        if locale_name is not None:
907            locale_name = locale_name[0]
908
909    if locale_name is None:
910        return None, None
911
912    normalize = locale.normalize(locale_name)
913    if normalize == locale_name:
914        language = region = locale_name
915    else:
916        locale_name = normalize
917        if '.' in locale_name:
918            locale_name, _ = locale_name.split('.', 1)
919        language, region = locale_name.lower().split("_")
920    return language, region
921