1#!/usr/bin/env python
2#
3# Copyright 2011-2015 The Rust Project Developers. See the COPYRIGHT
4# file at the top-level directory of this distribution and at
5# http://rust-lang.org/COPYRIGHT.
6#
7# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
8# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
9# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
10# option. This file may not be copied, modified, or distributed
11# except according to those terms.
12
13# This script uses the following Unicode tables:
14# - DerivedCoreProperties.txt
15# - auxiliary/GraphemeBreakProperty.txt
16# - auxiliary/WordBreakProperty.txt
17# - ReadMe.txt
18# - UnicodeData.txt
19#
20# Since this should not require frequent updates, we just store this
21# out-of-line and check the unicode.rs file into git.
22
23import fileinput, re, os, sys
24
25preamble = '''// Copyright 2012-2018 The Rust Project Developers. See the COPYRIGHT
26// file at the top-level directory of this distribution and at
27// http://rust-lang.org/COPYRIGHT.
28//
29// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
30// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
31// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
32// option. This file may not be copied, modified, or distributed
33// except according to those terms.
34
35// NOTE: The following code was generated by "scripts/unicode.py", do not edit directly
36
37#![allow(missing_docs, non_upper_case_globals, non_snake_case)]
38
39use super::ScriptExtension;
40'''
41
42UNICODE_VERSION = (12, 0, 0)
43
44UNICODE_VERSION_NUMBER = "%s.%s.%s" %UNICODE_VERSION
45
46def escape_char(c):
47    return "'\\u{%x}'" % c
48
49def fetch(f):
50    if not os.path.exists(os.path.basename(f)):
51        if "emoji" in f:
52            os.system("curl -O https://www.unicode.org/Public/emoji/%s.%s/%s"
53                      % (UNICODE_VERSION[0], UNICODE_VERSION[1], f))
54        else:
55            os.system("curl -O http://www.unicode.org/Public/%s/ucd/%s"
56                      % (UNICODE_VERSION_NUMBER, f))
57
58    if not os.path.exists(os.path.basename(f)):
59        sys.stderr.write("cannot load %s" % f)
60        exit(1)
61
62def group_cats(cats):
63    cats_out = {}
64    for cat in cats:
65        cats_out[cat] = group_cat(cats[cat])
66    return cats_out
67
68def aliases():
69    """
70    Fetch the shorthand aliases for each longhand Script name
71    """
72    fetch("PropertyValueAliases.txt")
73    longforms = {}
74    shortforms = {}
75    re1 = re.compile(r"^ *sc *; *(\w+) *; *(\w+)")
76    for line in fileinput.input(os.path.basename("PropertyValueAliases.txt")):
77        m = re1.match(line)
78        if m:
79            l = m.group(2).strip()
80            s = m.group(1).strip()
81            assert(s not in longforms)
82            assert(l not in shortforms)
83            longforms[s] = l
84            shortforms[l] = s
85        else:
86            continue
87
88    return (longforms, shortforms)
89
90def format_table_content(f, content, indent):
91    line = " "*indent
92    first = True
93    for chunk in content.split(","):
94        if len(line) + len(chunk) < 98:
95            if first:
96                line += chunk
97            else:
98                line += ", " + chunk
99            first = False
100        else:
101            f.write(line + ",\n")
102            line = " "*indent + chunk
103    f.write(line)
104
105# Implementation from unicode-segmentation
106def load_properties(f, interestingprops):
107    fetch(f)
108    props = {}
109    # Note: these regexes are different from those in unicode-segmentation,
110    # becase we need to handle spaces here
111    re1 = re.compile(r"^ *([0-9A-F]+) *; *([^#]+) *#")
112    re2 = re.compile(r"^ *([0-9A-F]+)\.\.([0-9A-F]+) *; *([^#]+) *#")
113
114    for line in fileinput.input(os.path.basename(f)):
115        prop = None
116        d_lo = 0
117        d_hi = 0
118        m = re1.match(line)
119        if m:
120            d_lo = m.group(1)
121            d_hi = m.group(1)
122            prop = m.group(2).strip()
123        else:
124            m = re2.match(line)
125            if m:
126                d_lo = m.group(1)
127                d_hi = m.group(2)
128                prop = m.group(3).strip()
129            else:
130                continue
131        if interestingprops and prop not in interestingprops:
132            continue
133        d_lo = int(d_lo, 16)
134        d_hi = int(d_hi, 16)
135        if prop not in props:
136            props[prop] = []
137        props[prop].append((d_lo, d_hi))
138
139    return props
140
141# Implementation from unicode-segmentation
142def emit_table(f, name, t_data, t_type = "&'static [(char, char)]", is_pub=True,
143        pfun=lambda x: "(%s,%s)" % (escape_char(x[0]), escape_char(x[1])), is_const=True):
144    pub_string = "const"
145    if not is_const:
146        pub_string = "let"
147    if is_pub:
148        pub_string = "pub " + pub_string
149    f.write("    %s %s: %s = &[\n" % (pub_string, name, t_type))
150    data = ""
151    first = True
152    for dat in t_data:
153        if not first:
154            data += ","
155        first = False
156        data += pfun(dat)
157    format_table_content(f, data, 8)
158    f.write("\n    ];\n\n")
159
160def emit_search(f):
161    f.write("""
162pub fn bsearch_range_value_table<T: Copy>(c: char, r: &'static [(char, char, T)]) -> Option<T> {
163    use core::cmp::Ordering::{Equal, Less, Greater};
164    match r.binary_search_by(|&(lo, hi, _)| {
165        if lo <= c && c <= hi { Equal }
166        else if hi < c { Less }
167        else { Greater }
168    }) {
169        Ok(idx) => {
170            let (_, _, cat) = r[idx];
171            Some(cat)
172        }
173        Err(_) => None
174    }
175}
176
177#[inline]
178pub fn get_script(c: char) -> Option<Script> {
179    bsearch_range_value_table(c, SCRIPTS)
180}
181
182#[inline]
183pub fn get_script_extension(c: char) -> Option<ScriptExtension> {
184    bsearch_range_value_table(c, SCRIPT_EXTENSIONS)
185}
186""")
187
188def emit_enums(f, script_list, extension_list, longforms):
189    """
190    Emit the Script and ScriptExtension enums as well as any related utility functions
191    """
192
193    f.write("""
194#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)]
195#[non_exhaustive]
196#[allow(non_camel_case_types)]
197#[repr(u8)]
198/// A value of the `Script` property
199pub enum Script {
200    /// Unknown script
201    Unknown = 0xFF,
202    /// Zyyy
203    Common = 0xFE,
204    /// Zinh,
205    Inherited = 0xFD,
206""")
207    for (i, script) in enumerate(script_list):
208        f.write("    /// %s\n    %s = %s,\n" % (script, longforms[script], i))
209    f.write("}\n")
210    f.write("pub const NEXT_SCRIPT: u8 = %s;" % len(script_list))
211    f.write("""
212
213pub mod script_extensions {
214    use crate::ScriptExtension;
215    pub const COMMON: ScriptExtension = ScriptExtension::new_common();
216    pub const INHERITED: ScriptExtension = ScriptExtension::new_inherited();
217    pub const UNKNOWN: ScriptExtension = ScriptExtension::new_unknown();
218""")
219    for (i, script) in enumerate(script_list):
220        first = 0
221        second = 0
222        third = 0
223        # need to replace L because `hex()` will spit out an L suffix for larger numbers
224        if i < 64:
225            first = hex(1 << i).replace("L", "")
226        elif i < 128:
227            second = hex(1 << (i - 64)).replace("L", "")
228        else:
229            third = hex(1 << (i - 128)).replace("L", "")
230        f.write("    /// %s\n    pub const %s: ScriptExtension = ScriptExtension::new(%s, %s, %s);\n" %
231                (longforms[script], longforms[script].upper(), first, second, third))
232        if script != longforms[script]:
233            f.write("    /// %s\n    pub const %s: ScriptExtension = %s;\n" %
234                    (longforms[script], script.upper(), longforms[script].upper()))
235    for ext in extension_list:
236        longform = ", ".join([longforms[s] for s in ext])
237        name = "_".join([s.upper() for s in ext])
238        expr = ext[0].upper()
239        for e in ext[1:]:
240            expr = "%s.union(%s)" % (expr, e.upper())
241        f.write("    /// %s\n    pub const %s: ScriptExtension = %s;\n" % (longform, name, expr))
242    f.write("""}
243
244impl Script {
245    #[inline]
246    pub(crate) fn inner_full_name(self) -> &'static str {
247        match self {
248            Script::Unknown => "Unknown",
249            Script::Common => "Common",
250            Script::Inherited => "Inherited",
251""")
252    for script in script_list:
253        f.write("            Script::%s => \"%s\",\n" % (longforms[script], longforms[script]))
254    f.write("""        }
255    }
256
257    #[inline]
258    pub(crate) fn inner_short_name(self) -> &'static str {
259        match self {
260            Script::Unknown => "",
261            Script::Common => "Zyyy",
262            Script::Inherited => "Zinh",
263""")
264    for script in script_list:
265        f.write("            Script::%s => \"%s\",\n" % (longforms[script], script))
266    f.write("""        }
267    }
268
269    #[inline]
270    pub(crate) fn for_integer(value: u8) -> Self {
271        match value {
272""")
273    for (i, script) in enumerate(script_list):
274        f.write("            %s => Script::%s,\n" % (i, longforms[script]))
275    f.write("""            _ => unreachable!(),
276        }
277    }
278}
279""")
280
281def extension_name(ext):
282    """Get the rust source for a given ScriptExtension"""
283    return "script_extensions::%s" % "_".join([e.upper() for e in ext])
284
285
286
287
288if __name__ == "__main__":
289    r = "tables.rs"
290    if os.path.exists(r):
291        os.remove(r)
292    with open(r, "w") as rf:
293        # write the file's preamble
294        rf.write(preamble)
295        rf.write("""
296/// The version of [Unicode](http://www.unicode.org/)
297/// that this version of unicode-script is based on.
298pub const UNICODE_VERSION: (u64, u64, u64) = (%s, %s, %s);
299""" % UNICODE_VERSION)
300
301
302        (longforms, shortforms) = aliases()
303
304        scripts = load_properties("Scripts.txt", [])
305
306        script_table = []
307        script_list = []
308
309        for script in scripts:
310            if script not in ["Common", "Unknown", "Inherited"]:
311                script_list.append(shortforms[script])
312            script_table.extend([(x, y, shortforms[script]) for (x, y) in scripts[script]])
313        script_list.sort()
314        script_table.sort(key=lambda w: w[0])
315
316
317        extensions = load_properties("ScriptExtensions.txt", [])
318        extension_table = []
319        extension_list = []
320
321        for ext in extensions:
322            split = ext.split(" ")
323            split.sort()
324            output_ext = [ext]
325            if len(split) > 1:
326                extension_list.append(split)
327                output_ext = split
328            extension_table.extend([(x, y, output_ext) for (x, y) in extensions[ext]])
329        extension_table.sort(key=lambda w: w[0])
330
331
332        emit_enums(rf, script_list, extension_list, longforms)
333        emit_search(rf)
334
335        emit_table(rf, "SCRIPTS", script_table, t_type = "&'static [(char, char, Script)]",
336                   is_pub=False , pfun=lambda x: "(%s,%s, Script::%s)" % (escape_char(x[0]), escape_char(x[1]), longforms[x[2]]))
337        emit_table(rf, "SCRIPT_EXTENSIONS", extension_table, t_type = "&'static [(char, char, ScriptExtension)]",
338                   is_pub=False , pfun=lambda x: "(%s,%s,%s)" % (escape_char(x[0]), escape_char(x[1]), extension_name(x[2])))
339
340        # emit_table(rf, "FOObar", properties)
341