1 // Copyright 2015 The Servo Project Developers. See the
2 // COPYRIGHT file at the top-level directory of this distribution.
3 //
4 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
5 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
6 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
7 // option. This file may not be copied, modified, or distributed
8 // except according to those terms.
9
10 //! This crate implements the [Unicode Bidirectional Algorithm][tr9] for display of mixed
11 //! right-to-left and left-to-right text. It is written in safe Rust, compatible with the
12 //! current stable release.
13 //!
14 //! ## Example
15 //!
16 //! ```rust
17 //! use unicode_bidi::BidiInfo;
18 //!
19 //! // This example text is defined using `concat!` because some browsers
20 //! // and text editors have trouble displaying bidi strings.
21 //! let text = concat![
22 //! "א",
23 //! "ב",
24 //! "ג",
25 //! "a",
26 //! "b",
27 //! "c",
28 //! ];
29 //!
30 //! // Resolve embedding levels within the text. Pass `None` to detect the
31 //! // paragraph level automatically.
32 //! let bidi_info = BidiInfo::new(&text, None);
33 //!
34 //! // This paragraph has embedding level 1 because its first strong character is RTL.
35 //! assert_eq!(bidi_info.paragraphs.len(), 1);
36 //! let para = &bidi_info.paragraphs[0];
37 //! assert_eq!(para.level.number(), 1);
38 //! assert_eq!(para.level.is_rtl(), true);
39 //!
40 //! // Re-ordering is done after wrapping each paragraph into a sequence of
41 //! // lines. For this example, I'll just use a single line that spans the
42 //! // entire paragraph.
43 //! let line = para.range.clone();
44 //!
45 //! let display = bidi_info.reorder_line(para, line);
46 //! assert_eq!(display, concat![
47 //! "a",
48 //! "b",
49 //! "c",
50 //! "ג",
51 //! "ב",
52 //! "א",
53 //! ]);
54 //! ```
55 //!
56 //! # Features
57 //!
58 //! - `std`: Enabled by default, but can be disabled to make `unicode_bidi`
59 //! `#![no_std]` + `alloc` compatible.
60 //! - `serde`: Adds [`serde::Serialize`] and [`serde::Deserialize`]
61 //! implementations to relevant types.
62 //!
63 //! [tr9]: <http://www.unicode.org/reports/tr9/>
64
65 #![forbid(unsafe_code)]
66
67 #![no_std]
68 // We need to link to std to make doc tests work on older Rust versions
69 #[cfg(feature = "std")]
70 extern crate std;
71 #[macro_use]
72 extern crate alloc;
73
74 pub mod deprecated;
75 pub mod format_chars;
76 pub mod level;
77
78 mod char_data;
79 mod explicit;
80 mod implicit;
81 mod prepare;
82
83 pub use crate::char_data::{BidiClass, bidi_class, UNICODE_VERSION};
84 pub use crate::level::{Level, LTR_LEVEL, RTL_LEVEL};
85 pub use crate::prepare::LevelRun;
86
87 use alloc::borrow::Cow;
88 use alloc::vec::Vec;
89 use alloc::string::String;
90 use core::cmp::{max, min};
91 use core::iter::repeat;
92 use core::ops::Range;
93
94 use crate::BidiClass::*;
95 use crate::format_chars as chars;
96
97 /// Bidi information about a single paragraph
98 #[derive(Debug, PartialEq)]
99 pub struct ParagraphInfo {
100 /// The paragraphs boundaries within the text, as byte indices.
101 ///
102 /// TODO: Shrink this to only include the starting index?
103 pub range: Range<usize>,
104
105 /// The paragraph embedding level.
106 ///
107 /// <http://www.unicode.org/reports/tr9/#BD4>
108 pub level: Level,
109 }
110
111 /// Initial bidi information of the text.
112 ///
113 /// Contains the text paragraphs and `BidiClass` of its characters.
114 #[derive(PartialEq, Debug)]
115 pub struct InitialInfo<'text> {
116 /// The text
117 pub text: &'text str,
118
119 /// The BidiClass of the character at each byte in the text.
120 /// If a character is multiple bytes, its class will appear multiple times in the vector.
121 pub original_classes: Vec<BidiClass>,
122
123 /// The boundaries and level of each paragraph within the text.
124 pub paragraphs: Vec<ParagraphInfo>,
125 }
126
127 impl<'text> InitialInfo<'text> {
128 /// Find the paragraphs and BidiClasses in a string of text.
129 ///
130 /// <http://www.unicode.org/reports/tr9/#The_Paragraph_Level>
131 ///
132 /// Also sets the class for each First Strong Isolate initiator (FSI) to LRI or RLI if a strong
133 /// character is found before the matching PDI. If no strong character is found, the class will
134 /// remain FSI, and it's up to later stages to treat these as LRI when needed.
135 #[cfg_attr(feature = "flame_it", flamer::flame)]
new(text: &str, default_para_level: Option<Level>) -> InitialInfo<'_>136 pub fn new(text: &str, default_para_level: Option<Level>) -> InitialInfo<'_> {
137 let mut original_classes = Vec::with_capacity(text.len());
138
139 // The stack contains the starting byte index for each nested isolate we're inside.
140 let mut isolate_stack = Vec::new();
141 let mut paragraphs = Vec::new();
142
143 let mut para_start = 0;
144 let mut para_level = default_para_level;
145
146 #[cfg(feature = "flame_it")] flame::start("InitialInfo::new(): iter text.char_indices()");
147
148 for (i, c) in text.char_indices() {
149 let class = bidi_class(c);
150
151 #[cfg(feature = "flame_it")] flame::start("original_classes.extend()");
152
153 original_classes.extend(repeat(class).take(c.len_utf8()));
154
155 #[cfg(feature = "flame_it")] flame::end("original_classes.extend()");
156
157 match class {
158
159 B => {
160 // P1. Split the text into separate paragraphs. The paragraph separator is kept
161 // with the previous paragraph.
162 let para_end = i + c.len_utf8();
163 paragraphs.push(ParagraphInfo {
164 range: para_start..para_end,
165 // P3. If no character is found in p2, set the paragraph level to zero.
166 level: para_level.unwrap_or(LTR_LEVEL),
167 });
168 // Reset state for the start of the next paragraph.
169 para_start = para_end;
170 // TODO: Support defaulting to direction of previous paragraph
171 //
172 // <http://www.unicode.org/reports/tr9/#HL1>
173 para_level = default_para_level;
174 isolate_stack.clear();
175 }
176
177 L | R | AL => {
178 match isolate_stack.last() {
179 Some(&start) => {
180 if original_classes[start] == FSI {
181 // X5c. If the first strong character between FSI and its matching
182 // PDI is R or AL, treat it as RLI. Otherwise, treat it as LRI.
183 for j in 0..chars::FSI.len_utf8() {
184 original_classes[start + j] =
185 if class == L { LRI } else { RLI };
186 }
187 }
188 }
189
190 None => {
191 if para_level.is_none() {
192 // P2. Find the first character of type L, AL, or R, while skipping
193 // any characters between an isolate initiator and its matching
194 // PDI.
195 para_level = Some(if class != L { RTL_LEVEL } else { LTR_LEVEL });
196 }
197 }
198 }
199 }
200
201 RLI | LRI | FSI => {
202 isolate_stack.push(i);
203 }
204
205 PDI => {
206 isolate_stack.pop();
207 }
208
209 _ => {}
210 }
211 }
212 if para_start < text.len() {
213 paragraphs.push(ParagraphInfo {
214 range: para_start..text.len(),
215 level: para_level.unwrap_or(LTR_LEVEL),
216 });
217 }
218 assert_eq!(original_classes.len(), text.len());
219
220 #[cfg(feature = "flame_it")] flame::end("InitialInfo::new(): iter text.char_indices()");
221
222 InitialInfo {
223 text,
224 original_classes,
225 paragraphs,
226 }
227 }
228 }
229
230 /// Bidi information of the text.
231 ///
232 /// The `original_classes` and `levels` vectors are indexed by byte offsets into the text. If a
233 /// character is multiple bytes wide, then its class and level will appear multiple times in these
234 /// vectors.
235 // TODO: Impl `struct StringProperty<T> { values: Vec<T> }` and use instead of Vec<T>
236 #[derive(Debug, PartialEq)]
237 pub struct BidiInfo<'text> {
238 /// The text
239 pub text: &'text str,
240
241 /// The BidiClass of the character at each byte in the text.
242 pub original_classes: Vec<BidiClass>,
243
244 /// The directional embedding level of each byte in the text.
245 pub levels: Vec<Level>,
246
247 /// The boundaries and paragraph embedding level of each paragraph within the text.
248 ///
249 /// TODO: Use SmallVec or similar to avoid overhead when there are only one or two paragraphs?
250 /// Or just don't include the first paragraph, which always starts at 0?
251 pub paragraphs: Vec<ParagraphInfo>,
252 }
253
254 impl<'text> BidiInfo<'text> {
255 /// Split the text into paragraphs and determine the bidi embedding levels for each paragraph.
256 ///
257 /// TODO: In early steps, check for special cases that allow later steps to be skipped. like
258 /// text that is entirely LTR. See the `nsBidi` class from Gecko for comparison.
259 ///
260 /// TODO: Support auto-RTL base direction
261 #[cfg_attr(feature = "flame_it", flamer::flame)]
new(text: &str, default_para_level: Option<Level>) -> BidiInfo<'_>262 pub fn new(text: &str, default_para_level: Option<Level>) -> BidiInfo<'_> {
263 let InitialInfo {
264 original_classes,
265 paragraphs,
266 ..
267 } = InitialInfo::new(text, default_para_level);
268
269 let mut levels = Vec::<Level>::with_capacity(text.len());
270 let mut processing_classes = original_classes.clone();
271
272 for para in ¶graphs {
273 let text = &text[para.range.clone()];
274 let original_classes = &original_classes[para.range.clone()];
275 let processing_classes = &mut processing_classes[para.range.clone()];
276
277 let new_len = levels.len() + para.range.len();
278 levels.resize(new_len, para.level);
279 let levels = &mut levels[para.range.clone()];
280
281 explicit::compute(
282 text,
283 para.level,
284 original_classes,
285 levels,
286 processing_classes,
287 );
288
289 let sequences = prepare::isolating_run_sequences(para.level, original_classes, levels);
290 for sequence in &sequences {
291 implicit::resolve_weak(sequence, processing_classes);
292 implicit::resolve_neutral(sequence, levels, processing_classes);
293 }
294 implicit::resolve_levels(processing_classes, levels);
295
296 assign_levels_to_removed_chars(para.level, original_classes, levels);
297 }
298
299 BidiInfo {
300 text,
301 original_classes,
302 paragraphs,
303 levels,
304 }
305 }
306
307 /// Re-order a line based on resolved levels and return only the embedding levels, one `Level`
308 /// per *byte*.
309 #[cfg_attr(feature = "flame_it", flamer::flame)]
reordered_levels(&self, para: &ParagraphInfo, line: Range<usize>) -> Vec<Level>310 pub fn reordered_levels(&self, para: &ParagraphInfo, line: Range<usize>) -> Vec<Level> {
311 let (levels, _) = self.visual_runs(para, line);
312 levels
313 }
314
315 /// Re-order a line based on resolved levels and return only the embedding levels, one `Level`
316 /// per *character*.
317 #[cfg_attr(feature = "flame_it", flamer::flame)]
reordered_levels_per_char( &self, para: &ParagraphInfo, line: Range<usize>, ) -> Vec<Level>318 pub fn reordered_levels_per_char(
319 &self,
320 para: &ParagraphInfo,
321 line: Range<usize>,
322 ) -> Vec<Level> {
323 let levels = self.reordered_levels(para, line);
324 self.text.char_indices().map(|(i, _)| levels[i]).collect()
325 }
326
327
328 /// Re-order a line based on resolved levels and return the line in display order.
329 #[cfg_attr(feature = "flame_it", flamer::flame)]
reorder_line(&self, para: &ParagraphInfo, line: Range<usize>) -> Cow<'text, str>330 pub fn reorder_line(&self, para: &ParagraphInfo, line: Range<usize>) -> Cow<'text, str> {
331 let (levels, runs) = self.visual_runs(para, line.clone());
332
333 // If all isolating run sequences are LTR, no reordering is needed
334 if runs.iter().all(|run| levels[run.start].is_ltr()) {
335 return self.text[line].into();
336 }
337
338 let mut result = String::with_capacity(line.len());
339 for run in runs {
340 if levels[run.start].is_rtl() {
341 result.extend(self.text[run].chars().rev());
342 } else {
343 result.push_str(&self.text[run]);
344 }
345 }
346 result.into()
347 }
348
349 /// Find the level runs within a line and return them in visual order.
350 ///
351 /// `line` is a range of bytes indices within `levels`.
352 ///
353 /// <http://www.unicode.org/reports/tr9/#Reordering_Resolved_Levels>
354 #[cfg_attr(feature = "flame_it", flamer::flame)]
visual_runs( &self, para: &ParagraphInfo, line: Range<usize>, ) -> (Vec<Level>, Vec<LevelRun>)355 pub fn visual_runs(
356 &self,
357 para: &ParagraphInfo,
358 line: Range<usize>,
359 ) -> (Vec<Level>, Vec<LevelRun>) {
360 assert!(line.start <= self.levels.len());
361 assert!(line.end <= self.levels.len());
362
363 let mut levels = self.levels.clone();
364 let line_classes = &self.original_classes[line.clone()];
365 let line_levels = &mut levels[line.clone()];
366
367 // Reset some whitespace chars to paragraph level.
368 // <http://www.unicode.org/reports/tr9/#L1>
369 let line_str: &str = &self.text[line.clone()];
370 let mut reset_from: Option<usize> = Some(0);
371 let mut reset_to: Option<usize> = None;
372 for (i, c) in line_str.char_indices() {
373 match line_classes[i] {
374 // Ignored by X9
375 RLE | LRE | RLO | LRO | PDF | BN => {}
376 // Segment separator, Paragraph separator
377 B | S => {
378 assert_eq!(reset_to, None);
379 reset_to = Some(i + c.len_utf8());
380 if reset_from == None {
381 reset_from = Some(i);
382 }
383 }
384 // Whitespace, isolate formatting
385 WS | FSI | LRI | RLI | PDI => {
386 if reset_from == None {
387 reset_from = Some(i);
388 }
389 }
390 _ => {
391 reset_from = None;
392 }
393 }
394 if let (Some(from), Some(to)) = (reset_from, reset_to) {
395 for level in &mut line_levels[from..to] {
396 *level = para.level;
397 }
398 reset_from = None;
399 reset_to = None;
400 }
401 }
402 if let Some(from) = reset_from {
403 for level in &mut line_levels[from..] {
404 *level = para.level;
405 }
406 }
407
408 // Find consecutive level runs.
409 let mut runs = Vec::new();
410 let mut start = line.start;
411 let mut run_level = levels[start];
412 let mut min_level = run_level;
413 let mut max_level = run_level;
414
415 for (i, &new_level) in levels.iter().enumerate().take(line.end).skip(start + 1) {
416 if new_level != run_level {
417 // End of the previous run, start of a new one.
418 runs.push(start..i);
419 start = i;
420 run_level = new_level;
421 min_level = min(run_level, min_level);
422 max_level = max(run_level, max_level);
423 }
424 }
425 runs.push(start..line.end);
426
427 let run_count = runs.len();
428
429 // Re-order the odd runs.
430 // <http://www.unicode.org/reports/tr9/#L2>
431
432 // Stop at the lowest *odd* level.
433 min_level = min_level.new_lowest_ge_rtl().expect("Level error");
434
435 while max_level >= min_level {
436 // Look for the start of a sequence of consecutive runs of max_level or higher.
437 let mut seq_start = 0;
438 while seq_start < run_count {
439 if self.levels[runs[seq_start].start] < max_level {
440 seq_start += 1;
441 continue;
442 }
443
444 // Found the start of a sequence. Now find the end.
445 let mut seq_end = seq_start + 1;
446 while seq_end < run_count {
447 if self.levels[runs[seq_end].start] < max_level {
448 break;
449 }
450 seq_end += 1;
451 }
452
453 // Reverse the runs within this sequence.
454 runs[seq_start..seq_end].reverse();
455
456 seq_start = seq_end;
457 }
458 max_level.lower(1).expect(
459 "Lowering embedding level below zero",
460 );
461 }
462
463 (levels, runs)
464 }
465
466 /// If processed text has any computed RTL levels
467 ///
468 /// This information is usually used to skip re-ordering of text when no RTL level is present
469 #[inline]
has_rtl(&self) -> bool470 pub fn has_rtl(&self) -> bool {
471 level::has_rtl(&self.levels)
472 }
473 }
474
475 /// Assign levels to characters removed by rule X9.
476 ///
477 /// The levels assigned to these characters are not specified by the algorithm. This function
478 /// assigns each one the level of the previous character, to avoid breaking level runs.
479 #[cfg_attr(feature = "flame_it", flamer::flame)]
assign_levels_to_removed_chars(para_level: Level, classes: &[BidiClass], levels: &mut [Level])480 fn assign_levels_to_removed_chars(para_level: Level, classes: &[BidiClass], levels: &mut [Level]) {
481 for i in 0..levels.len() {
482 if prepare::removed_by_x9(classes[i]) {
483 levels[i] = if i > 0 { levels[i - 1] } else { para_level };
484 }
485 }
486 }
487
488
489 #[cfg(test)]
490 mod tests {
491 use super::*;
492
493 #[test]
test_initial_text_info()494 fn test_initial_text_info() {
495 let text = "a1";
496 assert_eq!(
497 InitialInfo::new(text, None),
498 InitialInfo {
499 text,
500 original_classes: vec![L, EN],
501 paragraphs: vec![
502 ParagraphInfo {
503 range: 0..2,
504 level: LTR_LEVEL,
505 },
506 ],
507 }
508 );
509
510 let text = "غ א";
511 assert_eq!(
512 InitialInfo::new(text, None),
513 InitialInfo {
514 text,
515 original_classes: vec![AL, AL, WS, R, R],
516 paragraphs: vec![
517 ParagraphInfo {
518 range: 0..5,
519 level: RTL_LEVEL,
520 },
521 ],
522 }
523 );
524
525 let text = "a\u{2029}b";
526 assert_eq!(
527 InitialInfo::new(text, None),
528 InitialInfo {
529 text,
530 original_classes: vec![L, B, B, B, L],
531 paragraphs: vec![
532 ParagraphInfo {
533 range: 0..4,
534 level: LTR_LEVEL,
535 },
536 ParagraphInfo {
537 range: 4..5,
538 level: LTR_LEVEL,
539 },
540 ],
541 }
542 );
543
544 let text = format!("{}א{}a", chars::FSI, chars::PDI);
545 assert_eq!(
546 InitialInfo::new(&text, None),
547 InitialInfo {
548 text: &text,
549 original_classes: vec![RLI, RLI, RLI, R, R, PDI, PDI, PDI, L],
550 paragraphs: vec![
551 ParagraphInfo {
552 range: 0..9,
553 level: LTR_LEVEL,
554 },
555 ],
556 }
557 );
558 }
559
560 #[test]
test_process_text()561 fn test_process_text() {
562 let text = "abc123";
563 assert_eq!(
564 BidiInfo::new(text, Some(LTR_LEVEL)),
565 BidiInfo {
566 text,
567 levels: Level::vec(&[0, 0, 0, 0, 0, 0]),
568 original_classes: vec![L, L, L, EN, EN, EN],
569 paragraphs: vec![
570 ParagraphInfo {
571 range: 0..6,
572 level: LTR_LEVEL,
573 },
574 ],
575 }
576 );
577
578 let text = "abc אבג";
579 assert_eq!(
580 BidiInfo::new(text, Some(LTR_LEVEL)),
581 BidiInfo {
582 text,
583 levels: Level::vec(&[0, 0, 0, 0, 1, 1, 1, 1, 1, 1]),
584 original_classes: vec![L, L, L, WS, R, R, R, R, R, R],
585 paragraphs: vec![
586 ParagraphInfo {
587 range: 0..10,
588 level: LTR_LEVEL,
589 },
590 ],
591 }
592 );
593 assert_eq!(
594 BidiInfo::new(text, Some(RTL_LEVEL)),
595 BidiInfo {
596 text,
597 levels: Level::vec(&[2, 2, 2, 1, 1, 1, 1, 1, 1, 1]),
598 original_classes: vec![L, L, L, WS, R, R, R, R, R, R],
599 paragraphs: vec![
600 ParagraphInfo {
601 range: 0..10,
602 level: RTL_LEVEL,
603 },
604 ],
605 }
606 );
607
608 let text = "אבג abc";
609 assert_eq!(
610 BidiInfo::new(text, Some(LTR_LEVEL)),
611 BidiInfo {
612 text,
613 levels: Level::vec(&[1, 1, 1, 1, 1, 1, 0, 0, 0, 0]),
614 original_classes: vec![R, R, R, R, R, R, WS, L, L, L],
615 paragraphs: vec![
616 ParagraphInfo {
617 range: 0..10,
618 level: LTR_LEVEL,
619 },
620 ],
621 }
622 );
623 assert_eq!(
624 BidiInfo::new(text, None),
625 BidiInfo {
626 text,
627 levels: Level::vec(&[1, 1, 1, 1, 1, 1, 1, 2, 2, 2]),
628 original_classes: vec![R, R, R, R, R, R, WS, L, L, L],
629 paragraphs: vec![
630 ParagraphInfo {
631 range: 0..10,
632 level: RTL_LEVEL,
633 },
634 ],
635 }
636 );
637
638 let text = "غ2ظ א2ג";
639 assert_eq!(
640 BidiInfo::new(text, Some(LTR_LEVEL)),
641 BidiInfo {
642 text,
643 levels: Level::vec(&[1, 1, 2, 1, 1, 1, 1, 1, 2, 1, 1]),
644 original_classes: vec![AL, AL, EN, AL, AL, WS, R, R, EN, R, R],
645 paragraphs: vec![
646 ParagraphInfo {
647 range: 0..11,
648 level: LTR_LEVEL,
649 },
650 ],
651 }
652 );
653
654 let text = "a א.\nג";
655 assert_eq!(
656 BidiInfo::new(text, None),
657 BidiInfo {
658 text,
659 original_classes: vec![L, WS, R, R, CS, B, R, R],
660 levels: Level::vec(&[0, 0, 1, 1, 0, 0, 1, 1]),
661 paragraphs: vec![
662 ParagraphInfo {
663 range: 0..6,
664 level: LTR_LEVEL,
665 },
666 ParagraphInfo {
667 range: 6..8,
668 level: RTL_LEVEL,
669 },
670 ],
671 }
672 );
673
674 // BidiTest:69635 (AL ET EN)
675 let bidi_info = BidiInfo::new("\u{060B}\u{20CF}\u{06F9}", None);
676 assert_eq!(bidi_info.original_classes, vec![AL, AL, ET, ET, ET, EN, EN]);
677 }
678
679 #[test]
test_bidi_info_has_rtl()680 fn test_bidi_info_has_rtl() {
681 // ASCII only
682 assert_eq!(BidiInfo::new("123", None).has_rtl(), false);
683 assert_eq!(BidiInfo::new("123", Some(LTR_LEVEL)).has_rtl(), false);
684 assert_eq!(BidiInfo::new("123", Some(RTL_LEVEL)).has_rtl(), false);
685 assert_eq!(BidiInfo::new("abc", None).has_rtl(), false);
686 assert_eq!(BidiInfo::new("abc", Some(LTR_LEVEL)).has_rtl(), false);
687 assert_eq!(BidiInfo::new("abc", Some(RTL_LEVEL)).has_rtl(), false);
688 assert_eq!(BidiInfo::new("abc 123", None).has_rtl(), false);
689 assert_eq!(BidiInfo::new("abc\n123", None).has_rtl(), false);
690
691 // With Hebrew
692 assert_eq!(BidiInfo::new("אבּג", None).has_rtl(), true);
693 assert_eq!(BidiInfo::new("אבּג", Some(LTR_LEVEL)).has_rtl(), true);
694 assert_eq!(BidiInfo::new("אבּג", Some(RTL_LEVEL)).has_rtl(), true);
695 assert_eq!(BidiInfo::new("abc אבּג", None).has_rtl(), true);
696 assert_eq!(BidiInfo::new("abc\nאבּג", None).has_rtl(), true);
697 assert_eq!(BidiInfo::new("אבּג abc", None).has_rtl(), true);
698 assert_eq!(BidiInfo::new("אבּג\nabc", None).has_rtl(), true);
699 assert_eq!(BidiInfo::new("אבּג 123", None).has_rtl(), true);
700 assert_eq!(BidiInfo::new("אבּג\n123", None).has_rtl(), true);
701 }
702
reorder_paras(text: &str) -> Vec<Cow<'_, str>>703 fn reorder_paras(text: &str) -> Vec<Cow<'_, str>> {
704 let bidi_info = BidiInfo::new(text, None);
705 bidi_info
706 .paragraphs
707 .iter()
708 .map(|para| bidi_info.reorder_line(para, para.range.clone()))
709 .collect()
710 }
711
712 #[test]
test_reorder_line()713 fn test_reorder_line() {
714 // Bidi_Class: L L L B L L L B L L L
715 assert_eq!(
716 reorder_paras("abc\ndef\nghi"),
717 vec!["abc\n", "def\n", "ghi"]
718 );
719
720 // Bidi_Class: L L EN B L L EN B L L EN
721 assert_eq!(
722 reorder_paras("ab1\nde2\ngh3"),
723 vec!["ab1\n", "de2\n", "gh3"]
724 );
725
726 // Bidi_Class: L L L B AL AL AL
727 assert_eq!(reorder_paras("abc\nابج"), vec!["abc\n", "جبا"]);
728
729 // Bidi_Class: AL AL AL B L L L
730 assert_eq!(reorder_paras("ابج\nabc"), vec!["\nجبا", "abc"]);
731
732 assert_eq!(reorder_paras("1.-2"), vec!["1.-2"]);
733 assert_eq!(reorder_paras("1-.2"), vec!["1-.2"]);
734 assert_eq!(reorder_paras("abc אבג"), vec!["abc גבא"]);
735
736 // Numbers being weak LTR characters, cannot reorder strong RTL
737 assert_eq!(reorder_paras("123 אבג"), vec!["גבא 123"]);
738
739 assert_eq!(reorder_paras("abc\u{202A}def"), vec!["abc\u{202A}def"]);
740
741 assert_eq!(
742 reorder_paras("abc\u{202A}def\u{202C}ghi"),
743 vec!["abc\u{202A}def\u{202C}ghi"]
744 );
745
746 assert_eq!(
747 reorder_paras("abc\u{2066}def\u{2069}ghi"),
748 vec!["abc\u{2066}def\u{2069}ghi"]
749 );
750
751 // Testing for RLE Character
752 assert_eq!(
753 reorder_paras("\u{202B}abc אבג\u{202C}"),
754 vec!["\u{202B}\u{202C}גבא abc"]
755 );
756
757 // Testing neutral characters
758 assert_eq!(reorder_paras("אבג? אבג"), vec!["גבא ?גבא"]);
759
760 // Testing neutral characters with special case
761 assert_eq!(reorder_paras("A אבג?"), vec!["A גבא?"]);
762
763 // Testing neutral characters with Implicit RTL Marker
764 assert_eq!(
765 reorder_paras("A אבג?\u{200F}"),
766 vec!["A \u{200F}?גבא"]
767 );
768 assert_eq!(reorder_paras("אבג abc"), vec!["abc גבא"]);
769 assert_eq!(
770 reorder_paras("abc\u{2067}.-\u{2069}ghi"),
771 vec!["abc\u{2067}-.\u{2069}ghi"]
772 );
773
774 assert_eq!(
775 reorder_paras("Hello, \u{2068}\u{202E}world\u{202C}\u{2069}!"),
776 vec!["Hello, \u{2068}\u{202E}\u{202C}dlrow\u{2069}!"]
777 );
778
779 // With mirrorable characters in RTL run
780 assert_eq!(reorder_paras("א(ב)ג."), vec![".ג)ב(א"]);
781
782 // With mirrorable characters on level boundry
783 assert_eq!(
784 reorder_paras("אב(גד[&ef].)gh"),
785 vec!["ef].)gh&[דג(בא"]
786 );
787 }
788
reordered_levels_for_paras(text: &str) -> Vec<Vec<Level>>789 fn reordered_levels_for_paras(text: &str) -> Vec<Vec<Level>> {
790 let bidi_info = BidiInfo::new(text, None);
791 bidi_info
792 .paragraphs
793 .iter()
794 .map(|para| bidi_info.reordered_levels(para, para.range.clone()))
795 .collect()
796 }
797
reordered_levels_per_char_for_paras(text: &str) -> Vec<Vec<Level>>798 fn reordered_levels_per_char_for_paras(text: &str) -> Vec<Vec<Level>> {
799 let bidi_info = BidiInfo::new(text, None);
800 bidi_info
801 .paragraphs
802 .iter()
803 .map(|para| {
804 bidi_info.reordered_levels_per_char(para, para.range.clone())
805 })
806 .collect()
807 }
808
809 #[test]
test_reordered_levels()810 fn test_reordered_levels() {
811
812 // BidiTest:946 (LRI PDI)
813 let text = "\u{2067}\u{2069}";
814 assert_eq!(
815 reordered_levels_for_paras(text),
816 vec![Level::vec(&[0, 0, 0, 0, 0, 0])]
817 );
818 assert_eq!(
819 reordered_levels_per_char_for_paras(text),
820 vec![Level::vec(&[0, 0])]
821 );
822
823 let text = "aa טֶ";
824 let bidi_info = BidiInfo::new(text, None);
825 assert_eq!(
826 bidi_info.reordered_levels(&bidi_info.paragraphs[0], 3..7),
827 Level::vec(&[0, 0, 0, 1, 1, 1, 1]),
828 )
829
830 /* TODO
831 /// BidiTest:69635 (AL ET EN)
832 let text = "\u{060B}\u{20CF}\u{06F9}";
833 assert_eq!(
834 reordered_levels_for_paras(text),
835 vec![Level::vec(&[1, 1, 1, 1, 1, 2, 2])]
836 );
837 assert_eq!(
838 reordered_levels_per_char_for_paras(text),
839 vec![Level::vec(&[1, 1, 2])]
840 );
841 */
842
843 /* TODO
844 // BidiTest:291284 (AN RLI PDF R)
845 assert_eq!(
846 reordered_levels_per_char_for_paras("\u{0605}\u{2067}\u{202C}\u{0590}"),
847 vec![&["2", "0", "x", "1"]]
848 );
849 */
850 }
851 }
852
853
854 #[cfg(all(feature = "serde", test))]
855 mod serde_tests {
856 use serde_test::{Token, assert_tokens};
857 use super::*;
858
859 #[test]
test_levels()860 fn test_levels() {
861 let text = "abc אבג";
862 let bidi_info = BidiInfo::new(text, None);
863 let levels = bidi_info.levels;
864 assert_eq!(text.as_bytes().len(), 10);
865 assert_eq!(levels.len(), 10);
866 assert_tokens(
867 &levels,
868 &[
869 Token::Seq { len: Some(10) },
870 Token::NewtypeStruct { name: "Level" },
871 Token::U8(0),
872 Token::NewtypeStruct { name: "Level" },
873 Token::U8(0),
874 Token::NewtypeStruct { name: "Level" },
875 Token::U8(0),
876 Token::NewtypeStruct { name: "Level" },
877 Token::U8(0),
878 Token::NewtypeStruct { name: "Level" },
879 Token::U8(1),
880 Token::NewtypeStruct { name: "Level" },
881 Token::U8(1),
882 Token::NewtypeStruct { name: "Level" },
883 Token::U8(1),
884 Token::NewtypeStruct { name: "Level" },
885 Token::U8(1),
886 Token::NewtypeStruct { name: "Level" },
887 Token::U8(1),
888 Token::NewtypeStruct { name: "Level" },
889 Token::U8(1),
890 Token::SeqEnd,
891 ],
892 );
893 }
894 }
895