1#!/usr/bin/env python 2# 3# Copyright 2011-2015 The Rust Project Developers. See the COPYRIGHT 4# file at the top-level directory of this distribution and at 5# http://rust-lang.org/COPYRIGHT. 6# 7# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or 8# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license 9# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your 10# option. This file may not be copied, modified, or distributed 11# except according to those terms. 12 13# This script uses the following Unicode tables: 14# - DerivedCoreProperties.txt 15# - auxiliary/GraphemeBreakProperty.txt 16# - auxiliary/WordBreakProperty.txt 17# - ReadMe.txt 18# - UnicodeData.txt 19# 20# Since this should not require frequent updates, we just store this 21# out-of-line and check the unicode.rs file into git. 22 23import fileinput, re, os, sys 24 25preamble = '''// Copyright 2012-2018 The Rust Project Developers. See the COPYRIGHT 26// file at the top-level directory of this distribution and at 27// http://rust-lang.org/COPYRIGHT. 28// 29// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or 30// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license 31// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your 32// option. This file may not be copied, modified, or distributed 33// except according to those terms. 34 35// NOTE: The following code was generated by "scripts/unicode.py", do not edit directly 36 37#![allow(missing_docs, non_upper_case_globals, non_snake_case)] 38 39use super::ScriptExtension; 40''' 41 42UNICODE_VERSION = (12, 0, 0) 43 44UNICODE_VERSION_NUMBER = "%s.%s.%s" %UNICODE_VERSION 45 46def escape_char(c): 47 return "'\\u{%x}'" % c 48 49def fetch(f): 50 if not os.path.exists(os.path.basename(f)): 51 if "emoji" in f: 52 os.system("curl -O https://www.unicode.org/Public/emoji/%s.%s/%s" 53 % (UNICODE_VERSION[0], UNICODE_VERSION[1], f)) 54 else: 55 os.system("curl -O http://www.unicode.org/Public/%s/ucd/%s" 56 % (UNICODE_VERSION_NUMBER, f)) 57 58 if not os.path.exists(os.path.basename(f)): 59 sys.stderr.write("cannot load %s" % f) 60 exit(1) 61 62def group_cats(cats): 63 cats_out = {} 64 for cat in cats: 65 cats_out[cat] = group_cat(cats[cat]) 66 return cats_out 67 68def aliases(): 69 """ 70 Fetch the shorthand aliases for each longhand Script name 71 """ 72 fetch("PropertyValueAliases.txt") 73 longforms = {} 74 shortforms = {} 75 re1 = re.compile(r"^ *sc *; *(\w+) *; *(\w+)") 76 for line in fileinput.input(os.path.basename("PropertyValueAliases.txt")): 77 m = re1.match(line) 78 if m: 79 l = m.group(2).strip() 80 s = m.group(1).strip() 81 assert(s not in longforms) 82 assert(l not in shortforms) 83 longforms[s] = l 84 shortforms[l] = s 85 else: 86 continue 87 88 return (longforms, shortforms) 89 90def format_table_content(f, content, indent): 91 line = " "*indent 92 first = True 93 for chunk in content.split(","): 94 if len(line) + len(chunk) < 98: 95 if first: 96 line += chunk 97 else: 98 line += ", " + chunk 99 first = False 100 else: 101 f.write(line + ",\n") 102 line = " "*indent + chunk 103 f.write(line) 104 105# Implementation from unicode-segmentation 106def load_properties(f, interestingprops): 107 fetch(f) 108 props = {} 109 # Note: these regexes are different from those in unicode-segmentation, 110 # becase we need to handle spaces here 111 re1 = re.compile(r"^ *([0-9A-F]+) *; *([^#]+) *#") 112 re2 = re.compile(r"^ *([0-9A-F]+)\.\.([0-9A-F]+) *; *([^#]+) *#") 113 114 for line in fileinput.input(os.path.basename(f)): 115 prop = None 116 d_lo = 0 117 d_hi = 0 118 m = re1.match(line) 119 if m: 120 d_lo = m.group(1) 121 d_hi = m.group(1) 122 prop = m.group(2).strip() 123 else: 124 m = re2.match(line) 125 if m: 126 d_lo = m.group(1) 127 d_hi = m.group(2) 128 prop = m.group(3).strip() 129 else: 130 continue 131 if interestingprops and prop not in interestingprops: 132 continue 133 d_lo = int(d_lo, 16) 134 d_hi = int(d_hi, 16) 135 if prop not in props: 136 props[prop] = [] 137 props[prop].append((d_lo, d_hi)) 138 139 return props 140 141# Implementation from unicode-segmentation 142def emit_table(f, name, t_data, t_type = "&'static [(char, char)]", is_pub=True, 143 pfun=lambda x: "(%s,%s)" % (escape_char(x[0]), escape_char(x[1])), is_const=True): 144 pub_string = "const" 145 if not is_const: 146 pub_string = "let" 147 if is_pub: 148 pub_string = "pub " + pub_string 149 f.write(" %s %s: %s = &[\n" % (pub_string, name, t_type)) 150 data = "" 151 first = True 152 for dat in t_data: 153 if not first: 154 data += "," 155 first = False 156 data += pfun(dat) 157 format_table_content(f, data, 8) 158 f.write("\n ];\n\n") 159 160def emit_search(f): 161 f.write(""" 162pub fn bsearch_range_value_table<T: Copy>(c: char, r: &'static [(char, char, T)]) -> Option<T> { 163 use core::cmp::Ordering::{Equal, Less, Greater}; 164 match r.binary_search_by(|&(lo, hi, _)| { 165 if lo <= c && c <= hi { Equal } 166 else if hi < c { Less } 167 else { Greater } 168 }) { 169 Ok(idx) => { 170 let (_, _, cat) = r[idx]; 171 Some(cat) 172 } 173 Err(_) => None 174 } 175} 176 177#[inline] 178pub fn get_script(c: char) -> Option<Script> { 179 bsearch_range_value_table(c, SCRIPTS) 180} 181 182#[inline] 183pub fn get_script_extension(c: char) -> Option<ScriptExtension> { 184 bsearch_range_value_table(c, SCRIPT_EXTENSIONS) 185} 186""") 187 188def emit_enums(f, script_list, extension_list, longforms): 189 """ 190 Emit the Script and ScriptExtension enums as well as any related utility functions 191 """ 192 193 f.write(""" 194#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)] 195#[non_exhaustive] 196#[allow(non_camel_case_types)] 197#[repr(u8)] 198/// A value of the `Script` property 199pub enum Script { 200 /// Unknown script 201 Unknown = 0xFF, 202 /// Zyyy 203 Common = 0xFE, 204 /// Zinh, 205 Inherited = 0xFD, 206""") 207 for (i, script) in enumerate(script_list): 208 f.write(" /// %s\n %s = %s,\n" % (script, longforms[script], i)) 209 f.write("}\n") 210 f.write("pub const NEXT_SCRIPT: u8 = %s;" % len(script_list)) 211 f.write(""" 212 213pub mod script_extensions { 214 use crate::ScriptExtension; 215 pub const COMMON: ScriptExtension = ScriptExtension::new_common(); 216 pub const INHERITED: ScriptExtension = ScriptExtension::new_inherited(); 217 pub const UNKNOWN: ScriptExtension = ScriptExtension::new_unknown(); 218""") 219 for (i, script) in enumerate(script_list): 220 first = 0 221 second = 0 222 third = 0 223 # need to replace L because `hex()` will spit out an L suffix for larger numbers 224 if i < 64: 225 first = hex(1 << i).replace("L", "") 226 elif i < 128: 227 second = hex(1 << (i - 64)).replace("L", "") 228 else: 229 third = hex(1 << (i - 128)).replace("L", "") 230 f.write(" /// %s\n pub const %s: ScriptExtension = ScriptExtension::new(%s, %s, %s);\n" % 231 (longforms[script], longforms[script].upper(), first, second, third)) 232 if script != longforms[script]: 233 f.write(" /// %s\n pub const %s: ScriptExtension = %s;\n" % 234 (longforms[script], script.upper(), longforms[script].upper())) 235 for ext in extension_list: 236 longform = ", ".join([longforms[s] for s in ext]) 237 name = "_".join([s.upper() for s in ext]) 238 expr = ext[0].upper() 239 for e in ext[1:]: 240 expr = "%s.union(%s)" % (expr, e.upper()) 241 f.write(" /// %s\n pub const %s: ScriptExtension = %s;\n" % (longform, name, expr)) 242 f.write("""} 243 244impl Script { 245 #[inline] 246 pub(crate) fn inner_full_name(self) -> &'static str { 247 match self { 248 Script::Unknown => "Unknown", 249 Script::Common => "Common", 250 Script::Inherited => "Inherited", 251""") 252 for script in script_list: 253 f.write(" Script::%s => \"%s\",\n" % (longforms[script], longforms[script])) 254 f.write(""" } 255 } 256 257 #[inline] 258 pub(crate) fn inner_short_name(self) -> &'static str { 259 match self { 260 Script::Unknown => "", 261 Script::Common => "Zyyy", 262 Script::Inherited => "Zinh", 263""") 264 for script in script_list: 265 f.write(" Script::%s => \"%s\",\n" % (longforms[script], script)) 266 f.write(""" } 267 } 268 269 #[inline] 270 pub(crate) fn for_integer(value: u8) -> Self { 271 match value { 272""") 273 for (i, script) in enumerate(script_list): 274 f.write(" %s => Script::%s,\n" % (i, longforms[script])) 275 f.write(""" _ => unreachable!(), 276 } 277 } 278} 279""") 280 281def extension_name(ext): 282 """Get the rust source for a given ScriptExtension""" 283 return "script_extensions::%s" % "_".join([e.upper() for e in ext]) 284 285 286 287 288if __name__ == "__main__": 289 r = "tables.rs" 290 if os.path.exists(r): 291 os.remove(r) 292 with open(r, "w") as rf: 293 # write the file's preamble 294 rf.write(preamble) 295 rf.write(""" 296/// The version of [Unicode](http://www.unicode.org/) 297/// that this version of unicode-script is based on. 298pub const UNICODE_VERSION: (u64, u64, u64) = (%s, %s, %s); 299""" % UNICODE_VERSION) 300 301 302 (longforms, shortforms) = aliases() 303 304 scripts = load_properties("Scripts.txt", []) 305 306 script_table = [] 307 script_list = [] 308 309 for script in scripts: 310 if script not in ["Common", "Unknown", "Inherited"]: 311 script_list.append(shortforms[script]) 312 script_table.extend([(x, y, shortforms[script]) for (x, y) in scripts[script]]) 313 script_list.sort() 314 script_table.sort(key=lambda w: w[0]) 315 316 317 extensions = load_properties("ScriptExtensions.txt", []) 318 extension_table = [] 319 extension_list = [] 320 321 for ext in extensions: 322 split = ext.split(" ") 323 split.sort() 324 output_ext = [ext] 325 if len(split) > 1: 326 extension_list.append(split) 327 output_ext = split 328 extension_table.extend([(x, y, output_ext) for (x, y) in extensions[ext]]) 329 extension_table.sort(key=lambda w: w[0]) 330 331 332 emit_enums(rf, script_list, extension_list, longforms) 333 emit_search(rf) 334 335 emit_table(rf, "SCRIPTS", script_table, t_type = "&'static [(char, char, Script)]", 336 is_pub=False , pfun=lambda x: "(%s,%s, Script::%s)" % (escape_char(x[0]), escape_char(x[1]), longforms[x[2]])) 337 emit_table(rf, "SCRIPT_EXTENSIONS", extension_table, t_type = "&'static [(char, char, ScriptExtension)]", 338 is_pub=False , pfun=lambda x: "(%s,%s,%s)" % (escape_char(x[0]), escape_char(x[1]), extension_name(x[2]))) 339 340 # emit_table(rf, "FOObar", properties) 341