1#!/usr/bin/env python
2
3import os
4import sys
5import subprocess
6from pathlib import Path
7
8
9# There is no sane way to test them.
10IGNORE_TESTS = [
11    'macos.tests',
12]
13
14IGNORE_TEST_CASES = [
15    # aots tests
16    # Unknown issue. Investigate.
17    'gpos_context2_classes_001',
18    'gpos_context2_classes_002',
19
20    # in-house tests
21    # Unknown issue. Investigate.
22    'simple_002',
23    # Not possible to implement without shaping.
24    'arabic_fallback_shaping_001',
25
26    # text-rendering-tests tests
27    # Unknown issue. Investigate.
28    'cmap_1_004',
29    'morx_29_001',
30    'morx_29_002',
31    'morx_29_003',
32    'morx_29_004',
33    'morx_30_001',
34    'morx_30_002',
35    'morx_30_003',
36    'morx_30_004',
37    'morx_6_001',
38    'sharan_1_002',
39    'sharan_1_003',
40    'sharan_1_004',
41    'sharan_1_005',
42    'sharan_1_006',
43    'shknda_3_031',
44    'shlana_10_005',
45    'shlana_10_016',
46    'shlana_10_017',
47    'shlana_10_028',
48    'shlana_10_041',
49    'shlana_10_044',
50    'shlana_2_014',
51    'shlana_3_013',
52    'shlana_5_010',
53    'shlana_5_012',
54    'shlana_6_002',
55    'shlana_8_005',
56]
57
58
59def update_relative_path(tests_name, fontfile):
60    fontfile = fontfile.replace('../fonts/', '')
61    return f'tests/fonts/{tests_name}/{fontfile}'  # relative to the root dir
62
63
64# Converts `U+0041,U+0078` into `\u{0041}\u{0078}`
65def convert_unicodes(unicodes):
66    text = ''
67    for (i, u) in enumerate(unicodes.split(',')):
68        if i > 0 and i % 10 == 0:
69            text += '\\\n             '
70
71        text += f'\\u{{{u[2:]}}}'
72
73    return text
74
75
76def convert_test(hb_dir, hb_shape_exe, tests_name, file_name, idx, data, fonts):
77    fontfile, options, unicodes, glyphs_expected = data.split(':')
78
79    fontfile_rs = update_relative_path(tests_name, fontfile)
80
81    unicodes_rs = convert_unicodes(unicodes)
82
83    test_name = file_name.replace('.tests', '').replace('-', '_') + f'_{idx:03d}'
84    test_name = test_name.lower()
85
86    # We have to actually run hb-shape instead of using predefined results,
87    # because hb sometimes stores results for freetype and not for embedded OpenType
88    # engine, which we are using.
89    # Right now, it only affects 'text-rendering-tests'.
90    if len(options) != 0:
91        options_list = options.split(' ')
92    else:
93        options_list = []
94
95    options_list.insert(0, str(hb_shape_exe))
96
97    # Force OT functions, since this is the only one we support in rustybuzz.
98    options_list.append('--font-funcs=ot')
99    # Remove FT when present.
100    if '--font-funcs=ft' in options_list:
101        options_list.remove('--font-funcs=ft')
102
103    abs_font_path = hb_dir.joinpath('test/shaping/data')\
104        .joinpath(tests_name)\
105        .joinpath('tests') \
106        .joinpath(fontfile)
107
108    options_list.append(str(abs_font_path))
109    options_list.append(f'--unicodes={unicodes}')  # no need to escape it
110
111    glyphs_expected = subprocess.run(options_list, check=True, stdout=subprocess.PIPE)\
112        .stdout.decode()
113
114    glyphs_expected = glyphs_expected[1:-2]  # remove `[..]\n`
115    glyphs_expected = glyphs_expected.replace('|', '|\\\n         ')
116
117    options = options.replace('"', '\\"')
118
119    fonts.add(os.path.split(fontfile_rs)[1])
120
121    if test_name in IGNORE_TEST_CASES:
122        return ''
123
124    return (f'#[test]\n'
125            f'fn {test_name}() {{\n'
126            f'    assert_eq!(\n'
127            f'        shape(\n'
128            f'            "{fontfile_rs}",\n'
129            f'            "{unicodes_rs}",\n'
130            f'            "{options}",\n'
131            f'        ),\n'
132            f'        "{glyphs_expected}"\n'
133            f'    );\n'
134            f'}}\n'
135            '\n')
136
137
138def convert(hb_dir, hb_shape_exe, tests_dir, tests_name):
139    files = sorted(os.listdir(tests_dir))
140    files = [f for f in files if f.endswith('.tests')]
141
142    fonts = set()
143
144    rust_code = ('// WARNING: this file was generated by ../scripts/gen-shaping-tests.py\n'
145                 '\n'
146                 'mod shaping_impl;\n'
147                 'use shaping_impl::shape;\n'
148                 '\n')
149
150    for file in files:
151        if file in IGNORE_TESTS:
152            continue
153
154        with open(tests_dir / file, 'r') as f:
155            for idx, test in enumerate(f.read().splitlines()):
156                # skip comments and empty lines
157                if test.startswith('#') or len(test) == 0:
158                    continue
159
160                rust_code += convert_test(hb_dir, hb_shape_exe, tests_name,
161                                          file, idx + 1, test, fonts)
162
163    tests_name_snake_case = tests_name.replace('-', '_')
164    with open(f'../tests/shaping_{tests_name_snake_case}.rs', 'w') as f:
165        f.write(rust_code)
166
167    return fonts
168
169
170if len(sys.argv) != 2:
171    print('Usage: gen-shaping-tests.py /path/to/harfbuzz-src')
172    exit(1)
173
174hb_dir = Path(sys.argv[1])
175assert hb_dir.exists()
176
177# Check that harfbuzz was built.
178hb_shape_exe = hb_dir.joinpath('builddir/util/hb-shape')
179if not hb_shape_exe.exists():
180    print('Build harfbuzz first using:')
181    print('    meson builddir')
182    print('    ninja -Cbuilddir')
183    exit(1)
184
185used_fonts = []
186font_files = []
187test_dir_names = ['aots', 'in-house', 'text-rendering-tests']
188for test_dir_name in test_dir_names:
189    tests_dir = hb_dir / f'test/shaping/data/{test_dir_name}/tests'
190
191    used_fonts += convert(hb_dir, hb_shape_exe, tests_dir, test_dir_name)
192
193    font_files += os.listdir(hb_dir / f'test/shaping/data/{test_dir_name}/fonts')
194
195# Check for unused fonts.
196unused_fonts = sorted(list(set(font_files).difference(used_fonts)))
197if len(unused_fonts) != 0:
198    print('Unused fonts:')
199    for font in unused_fonts:
200        print(font)
201