1// Copyright 2015 The Servo Project Developers. See the
2// COPYRIGHT file at the top-level directory of this distribution.
3//
4// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
5// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
6// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
7// option. This file may not be copied, modified, or distributed
8// except according to those terms.
9
10//! This crate implements the [Unicode Bidirectional Algorithm][tr9] for display of mixed
11//! right-to-left and left-to-right text. It is written in safe Rust, compatible with the
12//! current stable release.
13//!
14//! ## Example
15//!
16//! ```rust
17//! # #[cfg(feature = "hardcoded-data")] {
18//! use unicode_bidi::BidiInfo;
19//!
20//! // This example text is defined using `concat!` because some browsers
21//! // and text editors have trouble displaying bidi strings.
22//! let text = concat![
23//! "א",
24//! "ב",
25//! "ג",
26//! "a",
27//! "b",
28//! "c",
29//! ];
30//!
31//! // Resolve embedding levels within the text. Pass `None` to detect the
32//! // paragraph level automatically.
33//! let bidi_info = BidiInfo::new(&text, None);
34//!
35//! // This paragraph has embedding level 1 because its first strong character is RTL.
36//! assert_eq!(bidi_info.paragraphs.len(), 1);
37//! let para = &bidi_info.paragraphs[0];
38//! assert_eq!(para.level.number(), 1);
39//! assert_eq!(para.level.is_rtl(), true);
40//!
41//! // Re-ordering is done after wrapping each paragraph into a sequence of
42//! // lines. For this example, I'll just use a single line that spans the
43//! // entire paragraph.
44//! let line = para.range.clone();
45//!
46//! let display = bidi_info.reorder_line(para, line);
47//! assert_eq!(display, concat![
48//! "a",
49//! "b",
50//! "c",
51//! "ג",
52//! "ב",
53//! "א",
54//! ]);
55//! # } // feature = "hardcoded-data"
56//! ```
57//!
58//! # Features
59//!
60//! - `std`: Enabled by default, but can be disabled to make `unicode_bidi`
61//! `#![no_std]` + `alloc` compatible.
62//! - `hardcoded-data`: Enabled by default. Includes hardcoded Unicode bidi data and more convenient APIs.
63//! - `serde`: Adds [`serde::Serialize`] and [`serde::Deserialize`]
64//! implementations to relevant types.
65//!
66//! [tr9]: <http://www.unicode.org/reports/tr9/>
67
68#![no_std]
69// We need to link to std to make doc tests work on older Rust versions
70#[cfg(feature = "std")]
71extern crate std;
72#[macro_use]
73extern crate alloc;
74
75pub mod data_source;
76pub mod deprecated;
77pub mod format_chars;
78pub mod level;
79
80mod char_data;
81mod explicit;
82mod implicit;
83mod prepare;
84
85pub use crate::char_data::{BidiClass, UNICODE_VERSION};
86pub use crate::data_source::BidiDataSource;
87pub use crate::level::{Level, LTR_LEVEL, RTL_LEVEL};
88pub use crate::prepare::LevelRun;
89
90#[cfg(feature = "hardcoded-data")]
91pub use crate::char_data::{bidi_class, HardcodedBidiData};
92
93use alloc::borrow::Cow;
94use alloc::string::String;
95use alloc::vec::Vec;
96use core::cmp;
97use core::iter::repeat;
98use core::ops::Range;
99
100use crate::format_chars as chars;
101use crate::BidiClass::*;
102
103#[derive(PartialEq, Debug)]
104pub enum Direction {
105 Ltr,
106 Rtl,
107 Mixed,
108}
109
110/// Bidi information about a single paragraph
111#[derive(Debug, PartialEq)]
112pub struct ParagraphInfo {
113 /// The paragraphs boundaries within the text, as byte indices.
114 ///
115 /// TODO: Shrink this to only include the starting index?
116 pub range: Range<usize>,
117
118 /// The paragraph embedding level.
119 ///
120 /// <http://www.unicode.org/reports/tr9/#BD4>
121 pub level: Level,
122}
123
124impl ParagraphInfo {
125 /// Gets the length of the paragraph in the source text.
126 pub fn len(&self) -> usize {
127 self.range.end - self.range.start
128 }
129}
130
131/// Initial bidi information of the text.
132///
133/// Contains the text paragraphs and `BidiClass` of its characters.
134#[derive(PartialEq, Debug)]
135pub struct InitialInfo<'text> {
136 /// The text
137 pub text: &'text str,
138
139 /// The BidiClass of the character at each byte in the text.
140 /// If a character is multiple bytes, its class will appear multiple times in the vector.
141 pub original_classes: Vec<BidiClass>,
142
143 /// The boundaries and level of each paragraph within the text.
144 pub paragraphs: Vec<ParagraphInfo>,
145}
146
147impl<'text> InitialInfo<'text> {
148 /// Find the paragraphs and BidiClasses in a string of text.
149 ///
150 /// <http://www.unicode.org/reports/tr9/#The_Paragraph_Level>
151 ///
152 /// Also sets the class for each First Strong Isolate initiator (FSI) to LRI or RLI if a strong
153 /// character is found before the matching PDI. If no strong character is found, the class will
154 /// remain FSI, and it's up to later stages to treat these as LRI when needed.
155 ///
156 /// The `hardcoded-data` Cargo feature (enabled by default) must be enabled to use this.
157 #[cfg_attr(feature = "flame_it", flamer::flame)]
158 #[cfg(feature = "hardcoded-data")]
159 pub fn new(text: &str, default_para_level: Option<Level>) -> InitialInfo<'_> {
160 Self::new_with_data_source(&HardcodedBidiData, text, default_para_level)
161 }
162
163 /// Find the paragraphs and BidiClasses in a string of text, with a custom [`BidiDataSource`]
164 /// for Bidi data. If you just wish to use the hardcoded Bidi data, please use [`InitialInfo::new()`]
165 /// instead (enabled with tbe default `hardcoded-data` Cargo feature)
166 ///
167 /// <http://www.unicode.org/reports/tr9/#The_Paragraph_Level>
168 ///
169 /// Also sets the class for each First Strong Isolate initiator (FSI) to LRI or RLI if a strong
170 /// character is found before the matching PDI. If no strong character is found, the class will
171 /// remain FSI, and it's up to later stages to treat these as LRI when needed.
172 #[cfg_attr(feature = "flame_it", flamer::flame)]
173 pub fn new_with_data_source<'a, D: BidiDataSource>(
174 data_source: &D,
175 text: &'a str,
176 default_para_level: Option<Level>,
177 ) -> InitialInfo<'a> {
178 let mut original_classes = Vec::with_capacity(text.len());
179
180 // The stack contains the starting byte index for each nested isolate we're inside.
181 let mut isolate_stack = Vec::new();
182 let mut paragraphs = Vec::new();
183
184 let mut para_start = 0;
185 let mut para_level = default_para_level;
186
187 #[cfg(feature = "flame_it")]
188 flame::start("InitialInfo::new(): iter text.char_indices()");
189
190 for (i, c) in text.char_indices() {
191 let class = data_source.bidi_class(c);
192
193 #[cfg(feature = "flame_it")]
194 flame::start("original_classes.extend()");
195
196 original_classes.extend(repeat(class).take(c.len_utf8()));
197
198 #[cfg(feature = "flame_it")]
199 flame::end("original_classes.extend()");
200
201 match class {
202 B => {
203 // P1. Split the text into separate paragraphs. The paragraph separator is kept
204 // with the previous paragraph.
205 let para_end = i + c.len_utf8();
206 paragraphs.push(ParagraphInfo {
207 range: para_start..para_end,
208 // P3. If no character is found in p2, set the paragraph level to zero.
209 level: para_level.unwrap_or(LTR_LEVEL),
210 });
211 // Reset state for the start of the next paragraph.
212 para_start = para_end;
213 // TODO: Support defaulting to direction of previous paragraph
214 //
215 // <http://www.unicode.org/reports/tr9/#HL1>
216 para_level = default_para_level;
217 isolate_stack.clear();
218 }
219
220 L | R | AL => {
221 match isolate_stack.last() {
222 Some(&start) => {
223 if original_classes[start] == FSI {
224 // X5c. If the first strong character between FSI and its matching
225 // PDI is R or AL, treat it as RLI. Otherwise, treat it as LRI.
226 for j in 0..chars::FSI.len_utf8() {
227 original_classes[start + j] =
228 if class == L { LRI } else { RLI };
229 }
230 }
231 }
232
233 None => {
234 if para_level.is_none() {
235 // P2. Find the first character of type L, AL, or R, while skipping
236 // any characters between an isolate initiator and its matching
237 // PDI.
238 para_level = Some(if class != L { RTL_LEVEL } else { LTR_LEVEL });
239 }
240 }
241 }
242 }
243
244 RLI | LRI | FSI => {
245 isolate_stack.push(i);
246 }
247
248 PDI => {
249 isolate_stack.pop();
250 }
251
252 _ => {}
253 }
254 }
255 if para_start < text.len() {
256 paragraphs.push(ParagraphInfo {
257 range: para_start..text.len(),
258 level: para_level.unwrap_or(LTR_LEVEL),
259 });
260 }
261 assert_eq!(original_classes.len(), text.len());
262
263 #[cfg(feature = "flame_it")]
264 flame::end("InitialInfo::new(): iter text.char_indices()");
265
266 InitialInfo {
267 text,
268 original_classes,
269 paragraphs,
270 }
271 }
272}
273
274/// Bidi information of the text.
275///
276/// The `original_classes` and `levels` vectors are indexed by byte offsets into the text. If a
277/// character is multiple bytes wide, then its class and level will appear multiple times in these
278/// vectors.
279// TODO: Impl `struct StringProperty<T> { values: Vec<T> }` and use instead of Vec<T>
280#[derive(Debug, PartialEq)]
281pub struct BidiInfo<'text> {
282 /// The text
283 pub text: &'text str,
284
285 /// The BidiClass of the character at each byte in the text.
286 pub original_classes: Vec<BidiClass>,
287
288 /// The directional embedding level of each byte in the text.
289 pub levels: Vec<Level>,
290
291 /// The boundaries and paragraph embedding level of each paragraph within the text.
292 ///
293 /// TODO: Use SmallVec or similar to avoid overhead when there are only one or two paragraphs?
294 /// Or just don't include the first paragraph, which always starts at 0?
295 pub paragraphs: Vec<ParagraphInfo>,
296}
297
298impl<'text> BidiInfo<'text> {
299 /// Split the text into paragraphs and determine the bidi embedding levels for each paragraph.
300 ///
301 ///
302 /// The `hardcoded-data` Cargo feature (enabled by default) must be enabled to use this.
303 ///
304 /// TODO: In early steps, check for special cases that allow later steps to be skipped. like
305 /// text that is entirely LTR. See the `nsBidi` class from Gecko for comparison.
306 ///
307 /// TODO: Support auto-RTL base direction
308 #[cfg_attr(feature = "flame_it", flamer::flame)]
309 #[cfg(feature = "hardcoded-data")]
310 pub fn new(text: &str, default_para_level: Option<Level>) -> BidiInfo<'_> {
311 Self::new_with_data_source(&HardcodedBidiData, text, default_para_level)
312 }
313
314 /// Split the text into paragraphs and determine the bidi embedding levels for each paragraph, with a custom [`BidiDataSource`]
315 /// for Bidi data. If you just wish to use the hardcoded Bidi data, please use [`BidiInfo::new()`]
316 /// instead (enabled with tbe default `hardcoded-data` Cargo feature).
317 ///
318 /// TODO: In early steps, check for special cases that allow later steps to be skipped. like
319 /// text that is entirely LTR. See the `nsBidi` class from Gecko for comparison.
320 ///
321 /// TODO: Support auto-RTL base direction
322 #[cfg_attr(feature = "flame_it", flamer::flame)]
323 pub fn new_with_data_source<'a, D: BidiDataSource>(
324 data_source: &D,
325 text: &'a str,
326 default_para_level: Option<Level>,
327 ) -> BidiInfo<'a> {
328 let InitialInfo {
329 original_classes,
330 paragraphs,
331 ..
332 } = InitialInfo::new_with_data_source(data_source, text, default_para_level);
333
334 let mut levels = Vec::<Level>::with_capacity(text.len());
335 let mut processing_classes = original_classes.clone();
336
337 for para in &paragraphs {
338 let text = &text[para.range.clone()];
339 let original_classes = &original_classes[para.range.clone()];
340 let processing_classes = &mut processing_classes[para.range.clone()];
341
342 let new_len = levels.len() + para.range.len();
343 levels.resize(new_len, para.level);
344 let levels = &mut levels[para.range.clone()];
345
346 explicit::compute(
347 text,
348 para.level,
349 original_classes,
350 levels,
351 processing_classes,
352 );
353
354 let sequences = prepare::isolating_run_sequences(para.level, original_classes, levels);
355 for sequence in &sequences {
356 implicit::resolve_weak(text, sequence, processing_classes);
357 implicit::resolve_neutral(
358 text,
359 data_source,
360 sequence,
361 levels,
362 original_classes,
363 processing_classes,
364 );
365 }
366 implicit::resolve_levels(processing_classes, levels);
367
368 assign_levels_to_removed_chars(para.level, original_classes, levels);
369 }
370
371 BidiInfo {
372 text,
373 original_classes,
374 paragraphs,
375 levels,
376 }
377 }
378
379 /// Re-order a line based on resolved levels and return only the embedding levels, one `Level`
380 /// per *byte*.
381 #[cfg_attr(feature = "flame_it", flamer::flame)]
382 pub fn reordered_levels(&self, para: &ParagraphInfo, line: Range<usize>) -> Vec<Level> {
383 let (levels, _) = self.visual_runs(para, line);
384 levels
385 }
386
387 /// Re-order a line based on resolved levels and return only the embedding levels, one `Level`
388 /// per *character*.
389 #[cfg_attr(feature = "flame_it", flamer::flame)]
390 pub fn reordered_levels_per_char(
391 &self,
392 para: &ParagraphInfo,
393 line: Range<usize>,
394 ) -> Vec<Level> {
395 let levels = self.reordered_levels(para, line);
396 self.text.char_indices().map(|(i, _)| levels[i]).collect()
397 }
398
399 /// Re-order a line based on resolved levels and return the line in display order.
400 #[cfg_attr(feature = "flame_it", flamer::flame)]
401 pub fn reorder_line(&self, para: &ParagraphInfo, line: Range<usize>) -> Cow<'text, str> {
402 let (levels, runs) = self.visual_runs(para, line.clone());
403
404 // If all isolating run sequences are LTR, no reordering is needed
405 if runs.iter().all(|run| levels[run.start].is_ltr()) {
406 return self.text[line].into();
407 }
408
409 let mut result = String::with_capacity(line.len());
410 for run in runs {
411 if levels[run.start].is_rtl() {
412 result.extend(self.text[run].chars().rev());
413 } else {
414 result.push_str(&self.text[run]);
415 }
416 }
417 result.into()
418 }
419
420 /// Reorders pre-calculated levels of a sequence of characters.
421 ///
422 /// NOTE: This is a convenience method that does not use a `Paragraph` object. It is
423 /// intended to be used when an application has determined the levels of the objects (character sequences)
424 /// and just needs to have them reordered.
425 ///
426 /// the index map will result in `indexMap[visualIndex]==logicalIndex`.
427 ///
428 /// This only runs [Rule L2](http://www.unicode.org/reports/tr9/#L2) as it does not have
429 /// information about the actual text.
430 ///
431 /// Furthermore, if `levels` is an array that is aligned with code units, bytes within a codepoint may be
432 /// reversed. You may need to fix up the map to deal with this. Alternatively, only pass in arrays where each `Level`
433 /// is for a single code point.
434 ///
435 ///
436 /// # # Example
437 /// ```
438 /// use unicode_bidi::BidiInfo;
439 /// use unicode_bidi::Level;
440 ///
441 /// let l0 = Level::from(0);
442 /// let l1 = Level::from(1);
443 /// let l2 = Level::from(2);
444 ///
445 /// let levels = vec![l0, l0, l0, l0];
446 /// let index_map = BidiInfo::reorder_visual(&levels);
447 /// assert_eq!(levels.len(), index_map.len());
448 /// assert_eq!(index_map, [0, 1, 2, 3]);
449 ///
450 /// let levels: Vec<Level> = vec![l0, l0, l0, l1, l1, l1, l2, l2];
451 /// let index_map = BidiInfo::reorder_visual(&levels);
452 /// assert_eq!(levels.len(), index_map.len());
453 /// assert_eq!(index_map, [0, 1, 2, 6, 7, 5, 4, 3]);
454 /// ```
455 pub fn reorder_visual(levels: &[Level]) -> Vec<usize> {
456 // Gets the next range of characters after start_index with a level greater
457 // than or equal to `max`
458 fn next_range(levels: &[level::Level], mut start_index: usize, max: Level) -> Range<usize> {
459 if levels.is_empty() || start_index >= levels.len() {
460 return start_index..start_index;
461 }
462 while let Some(l) = levels.get(start_index) {
463 if *l >= max {
464 break;
465 }
466 start_index += 1;
467 }
468
469 if levels.get(start_index).is_none() {
470 // If at the end of the array, adding one will
471 // produce an out-of-range end element
472 return start_index..start_index;
473 }
474
475 let mut end_index = start_index + 1;
476 while let Some(l) = levels.get(end_index) {
477 if *l < max {
478 return start_index..end_index;
479 }
480 end_index += 1;
481 }
482
483 start_index..end_index
484 }
485
486 // This implementation is similar to the L2 implementation in `visual_runs()`
487 // but it cannot benefit from a precalculated LevelRun vector so needs to be different.
488
489 if levels.is_empty() {
490 return vec![];
491 }
492
493 // Get the min and max levels
494 let (mut min, mut max) = levels
495 .iter()
496 .fold((levels[0], levels[0]), |(min, max), &l| {
497 (cmp::min(min, l), cmp::max(max, l))
498 });
499
500 // Initialize an index map
501 let mut result: Vec<usize> = (0..levels.len()).collect();
502
503 if min == max && min.is_ltr() {
504 // Everything is LTR and at the same level, do nothing
505 return result;
506 }
507
508 // Stop at the lowest *odd* level, since everything below that
509 // is LTR and does not need further reordering
510 min = min.new_lowest_ge_rtl().expect("Level error");
511
512 // For each max level, take all contiguous chunks of
513 // levels ≥ max and reverse them
514 //
515 // We can do this check with the original levels instead of checking reorderings because all
516 // prior reorderings will have been for contiguous chunks of levels >> max, which will
517 // be a subset of these chunks anyway.
518 while min <= max {
519 let mut range = 0..0;
520 loop {
521 range = next_range(levels, range.end, max);
522 result[range.clone()].reverse();
523
524 if range.end >= levels.len() {
525 break;
526 }
527 }
528
529 max.lower(1).expect("Level error");
530 }
531
532 result
533 }
534
535 /// Find the level runs within a line and return them in visual order.
536 ///
537 /// `line` is a range of bytes indices within `levels`.
538 ///
539 /// <http://www.unicode.org/reports/tr9/#Reordering_Resolved_Levels>
540 #[cfg_attr(feature = "flame_it", flamer::flame)]
541 pub fn visual_runs(
542 &self,
543 para: &ParagraphInfo,
544 line: Range<usize>,
545 ) -> (Vec<Level>, Vec<LevelRun>) {
546 assert!(line.start <= self.levels.len());
547 assert!(line.end <= self.levels.len());
548
549 let mut levels = self.levels.clone();
550 let line_classes = &self.original_classes[line.clone()];
551 let line_levels = &mut levels[line.clone()];
552
553 // Reset some whitespace chars to paragraph level.
554 // <http://www.unicode.org/reports/tr9/#L1>
555 let line_str: &str = &self.text[line.clone()];
556 let mut reset_from: Option<usize> = Some(0);
557 let mut reset_to: Option<usize> = None;
558 let mut prev_level = para.level;
559 for (i, c) in line_str.char_indices() {
560 match line_classes[i] {
561 // Segment separator, Paragraph separator
562 B | S => {
563 assert_eq!(reset_to, None);
564 reset_to = Some(i + c.len_utf8());
565 if reset_from == None {
566 reset_from = Some(i);
567 }
568 }
569 // Whitespace, isolate formatting
570 WS | FSI | LRI | RLI | PDI => {
571 if reset_from == None {
572 reset_from = Some(i);
573 }
574 }
575 // <https://www.unicode.org/reports/tr9/#Retaining_Explicit_Formatting_Characters>
576 // same as above + set the level
577 RLE | LRE | RLO | LRO | PDF | BN => {
578 if reset_from == None {
579 reset_from = Some(i);
580 }
581 // also set the level to previous
582 line_levels[i] = prev_level;
583 }
584 _ => {
585 reset_from = None;
586 }
587 }
588 if let (Some(from), Some(to)) = (reset_from, reset_to) {
589 for level in &mut line_levels[from..to] {
590 *level = para.level;
591 }
592 reset_from = None;
593 reset_to = None;
594 }
595 prev_level = line_levels[i];
596 }
597 if let Some(from) = reset_from {
598 for level in &mut line_levels[from..] {
599 *level = para.level;
600 }
601 }
602
603 // Find consecutive level runs.
604 let mut runs = Vec::new();
605 let mut start = line.start;
606 let mut run_level = levels[start];
607 let mut min_level = run_level;
608 let mut max_level = run_level;
609
610 for (i, &new_level) in levels.iter().enumerate().take(line.end).skip(start + 1) {
611 if new_level != run_level {
612 // End of the previous run, start of a new one.
613 runs.push(start..i);
614 start = i;
615 run_level = new_level;
616 min_level = cmp::min(run_level, min_level);
617 max_level = cmp::max(run_level, max_level);
618 }
619 }
620 runs.push(start..line.end);
621
622 let run_count = runs.len();
623
624 // Re-order the odd runs.
625 // <http://www.unicode.org/reports/tr9/#L2>
626
627 // Stop at the lowest *odd* level.
628 min_level = min_level.new_lowest_ge_rtl().expect("Level error");
629
630 // This loop goes through contiguous chunks of level runs that have a level
631 // ≥ max_level and reverses their contents, reducing max_level by 1 each time.
632 //
633 // It can do this check with the original levels instead of checking reorderings because all
634 // prior reorderings will have been for contiguous chunks of levels >> max, which will
635 // be a subset of these chunks anyway.
636 while max_level >= min_level {
637 // Look for the start of a sequence of consecutive runs of max_level or higher.
638 let mut seq_start = 0;
639 while seq_start < run_count {
640 if self.levels[runs[seq_start].start] < max_level {
641 seq_start += 1;
642 continue;
643 }
644
645 // Found the start of a sequence. Now find the end.
646 let mut seq_end = seq_start + 1;
647 while seq_end < run_count {
648 if self.levels[runs[seq_end].start] < max_level {
649 break;
650 }
651 seq_end += 1;
652 }
653
654 // Reverse the runs within this sequence.
655 runs[seq_start..seq_end].reverse();
656
657 seq_start = seq_end;
658 }
659 max_level
660 .lower(1)
661 .expect("Lowering embedding level below zero");
662 }
663
664 (levels, runs)
665 }
666
667 /// If processed text has any computed RTL levels
668 ///
669 /// This information is usually used to skip re-ordering of text when no RTL level is present
670 #[inline]
671 pub fn has_rtl(&self) -> bool {
672 level::has_rtl(&self.levels)
673 }
674}
675
676/// Contains a reference of `BidiInfo` and one of its `paragraphs`.
677/// And it supports all operation in the `Paragraph` that needs also its
678/// `BidiInfo` such as `direction`.
679#[derive(Debug)]
680pub struct Paragraph<'a, 'text> {
681 pub info: &'a BidiInfo<'text>,
682 pub para: &'a ParagraphInfo,
683}
684
685impl<'a, 'text> Paragraph<'a, 'text> {
686 pub fn new(info: &'a BidiInfo<'text>, para: &'a ParagraphInfo) -> Paragraph<'a, 'text> {
687 Paragraph { info, para }
688 }
689
690 /// Returns if the paragraph is Left direction, right direction or mixed.
691 pub fn direction(&self) -> Direction {
692 let mut ltr = false;
693 let mut rtl = false;
694 for i in self.para.range.clone() {
695 if self.info.levels[i].is_ltr() {
696 ltr = true;
697 }
698
699 if self.info.levels[i].is_rtl() {
700 rtl = true;
701 }
702 }
703
704 if ltr && rtl {
705 return Direction::Mixed;
706 }
707
708 if ltr {
709 return Direction::Ltr;
710 }
711
712 Direction::Rtl
713 }
714
715 /// Returns the `Level` of a certain character in the paragraph.
716 pub fn level_at(&self, pos: usize) -> Level {
717 let actual_position = self.para.range.start + pos;
718 self.info.levels[actual_position]
719 }
720}
721
722/// Assign levels to characters removed by rule X9.
723///
724/// The levels assigned to these characters are not specified by the algorithm. This function
725/// assigns each one the level of the previous character, to avoid breaking level runs.
726#[cfg_attr(feature = "flame_it", flamer::flame)]
727fn assign_levels_to_removed_chars(para_level: Level, classes: &[BidiClass], levels: &mut [Level]) {
728 for i: usize in 0..levels.len() {
729 if prepare::removed_by_x9(class:classes[i]) {
730 levels[i] = if i > 0 { levels[i - 1] } else { para_level };
731 }
732 }
733}
734
735#[cfg(test)]
736#[cfg(feature = "hardcoded-data")]
737mod tests {
738 use super::*;
739
740 #[test]
741 fn test_initial_text_info() {
742 let text = "a1";
743 assert_eq!(
744 InitialInfo::new(text, None),
745 InitialInfo {
746 text,
747 original_classes: vec![L, EN],
748 paragraphs: vec![ParagraphInfo {
749 range: 0..2,
750 level: LTR_LEVEL,
751 },],
752 }
753 );
754
755 let text = "غ א";
756 assert_eq!(
757 InitialInfo::new(text, None),
758 InitialInfo {
759 text,
760 original_classes: vec![AL, AL, WS, R, R],
761 paragraphs: vec![ParagraphInfo {
762 range: 0..5,
763 level: RTL_LEVEL,
764 },],
765 }
766 );
767
768 let text = "a\u{2029}b";
769 assert_eq!(
770 InitialInfo::new(text, None),
771 InitialInfo {
772 text,
773 original_classes: vec![L, B, B, B, L],
774 paragraphs: vec![
775 ParagraphInfo {
776 range: 0..4,
777 level: LTR_LEVEL,
778 },
779 ParagraphInfo {
780 range: 4..5,
781 level: LTR_LEVEL,
782 },
783 ],
784 }
785 );
786
787 let text = format!("{}א{}a", chars::FSI, chars::PDI);
788 assert_eq!(
789 InitialInfo::new(&text, None),
790 InitialInfo {
791 text: &text,
792 original_classes: vec![RLI, RLI, RLI, R, R, PDI, PDI, PDI, L],
793 paragraphs: vec![ParagraphInfo {
794 range: 0..9,
795 level: LTR_LEVEL,
796 },],
797 }
798 );
799 }
800
801 #[test]
802 #[cfg(feature = "hardcoded-data")]
803 fn test_process_text() {
804 let text = "abc123";
805 assert_eq!(
806 BidiInfo::new(text, Some(LTR_LEVEL)),
807 BidiInfo {
808 text,
809 levels: Level::vec(&[0, 0, 0, 0, 0, 0]),
810 original_classes: vec![L, L, L, EN, EN, EN],
811 paragraphs: vec![ParagraphInfo {
812 range: 0..6,
813 level: LTR_LEVEL,
814 },],
815 }
816 );
817
818 let text = "abc אבג";
819 assert_eq!(
820 BidiInfo::new(text, Some(LTR_LEVEL)),
821 BidiInfo {
822 text,
823 levels: Level::vec(&[0, 0, 0, 0, 1, 1, 1, 1, 1, 1]),
824 original_classes: vec![L, L, L, WS, R, R, R, R, R, R],
825 paragraphs: vec![ParagraphInfo {
826 range: 0..10,
827 level: LTR_LEVEL,
828 },],
829 }
830 );
831 assert_eq!(
832 BidiInfo::new(text, Some(RTL_LEVEL)),
833 BidiInfo {
834 text,
835 levels: Level::vec(&[2, 2, 2, 1, 1, 1, 1, 1, 1, 1]),
836 original_classes: vec![L, L, L, WS, R, R, R, R, R, R],
837 paragraphs: vec![ParagraphInfo {
838 range: 0..10,
839 level: RTL_LEVEL,
840 },],
841 }
842 );
843
844 let text = "אבג abc";
845 assert_eq!(
846 BidiInfo::new(text, Some(LTR_LEVEL)),
847 BidiInfo {
848 text,
849 levels: Level::vec(&[1, 1, 1, 1, 1, 1, 0, 0, 0, 0]),
850 original_classes: vec![R, R, R, R, R, R, WS, L, L, L],
851 paragraphs: vec![ParagraphInfo {
852 range: 0..10,
853 level: LTR_LEVEL,
854 },],
855 }
856 );
857 assert_eq!(
858 BidiInfo::new(text, None),
859 BidiInfo {
860 text,
861 levels: Level::vec(&[1, 1, 1, 1, 1, 1, 1, 2, 2, 2]),
862 original_classes: vec![R, R, R, R, R, R, WS, L, L, L],
863 paragraphs: vec![ParagraphInfo {
864 range: 0..10,
865 level: RTL_LEVEL,
866 },],
867 }
868 );
869
870 let text = "غ2ظ א2ג";
871 assert_eq!(
872 BidiInfo::new(text, Some(LTR_LEVEL)),
873 BidiInfo {
874 text,
875 levels: Level::vec(&[1, 1, 2, 1, 1, 1, 1, 1, 2, 1, 1]),
876 original_classes: vec![AL, AL, EN, AL, AL, WS, R, R, EN, R, R],
877 paragraphs: vec![ParagraphInfo {
878 range: 0..11,
879 level: LTR_LEVEL,
880 },],
881 }
882 );
883
884 let text = "a א.\nג";
885 assert_eq!(
886 BidiInfo::new(text, None),
887 BidiInfo {
888 text,
889 original_classes: vec![L, WS, R, R, CS, B, R, R],
890 levels: Level::vec(&[0, 0, 1, 1, 0, 0, 1, 1]),
891 paragraphs: vec![
892 ParagraphInfo {
893 range: 0..6,
894 level: LTR_LEVEL,
895 },
896 ParagraphInfo {
897 range: 6..8,
898 level: RTL_LEVEL,
899 },
900 ],
901 }
902 );
903
904 // BidiTest:69635 (AL ET EN)
905 let bidi_info = BidiInfo::new("\u{060B}\u{20CF}\u{06F9}", None);
906 assert_eq!(bidi_info.original_classes, vec![AL, AL, ET, ET, ET, EN, EN]);
907 }
908
909 #[test]
910 #[cfg(feature = "hardcoded-data")]
911 fn test_bidi_info_has_rtl() {
912 // ASCII only
913 assert_eq!(BidiInfo::new("123", None).has_rtl(), false);
914 assert_eq!(BidiInfo::new("123", Some(LTR_LEVEL)).has_rtl(), false);
915 assert_eq!(BidiInfo::new("123", Some(RTL_LEVEL)).has_rtl(), false);
916 assert_eq!(BidiInfo::new("abc", None).has_rtl(), false);
917 assert_eq!(BidiInfo::new("abc", Some(LTR_LEVEL)).has_rtl(), false);
918 assert_eq!(BidiInfo::new("abc", Some(RTL_LEVEL)).has_rtl(), false);
919 assert_eq!(BidiInfo::new("abc 123", None).has_rtl(), false);
920 assert_eq!(BidiInfo::new("abc\n123", None).has_rtl(), false);
921
922 // With Hebrew
923 assert_eq!(BidiInfo::new("אבּג", None).has_rtl(), true);
924 assert_eq!(BidiInfo::new("אבּג", Some(LTR_LEVEL)).has_rtl(), true);
925 assert_eq!(BidiInfo::new("אבּג", Some(RTL_LEVEL)).has_rtl(), true);
926 assert_eq!(BidiInfo::new("abc אבּג", None).has_rtl(), true);
927 assert_eq!(BidiInfo::new("abc\nאבּג", None).has_rtl(), true);
928 assert_eq!(BidiInfo::new("אבּג abc", None).has_rtl(), true);
929 assert_eq!(BidiInfo::new("אבּג\nabc", None).has_rtl(), true);
930 assert_eq!(BidiInfo::new("אבּג 123", None).has_rtl(), true);
931 assert_eq!(BidiInfo::new("אבּג\n123", None).has_rtl(), true);
932 }
933
934 #[cfg(feature = "hardcoded-data")]
935 fn reorder_paras(text: &str) -> Vec<Cow<'_, str>> {
936 let bidi_info = BidiInfo::new(text, None);
937 bidi_info
938 .paragraphs
939 .iter()
940 .map(|para| bidi_info.reorder_line(para, para.range.clone()))
941 .collect()
942 }
943
944 #[test]
945 #[cfg(feature = "hardcoded-data")]
946 fn test_reorder_line() {
947 // Bidi_Class: L L L B L L L B L L L
948 assert_eq!(
949 reorder_paras("abc\ndef\nghi"),
950 vec!["abc\n", "def\n", "ghi"]
951 );
952
953 // Bidi_Class: L L EN B L L EN B L L EN
954 assert_eq!(
955 reorder_paras("ab1\nde2\ngh3"),
956 vec!["ab1\n", "de2\n", "gh3"]
957 );
958
959 // Bidi_Class: L L L B AL AL AL
960 assert_eq!(reorder_paras("abc\nابج"), vec!["abc\n", "جبا"]);
961
962 // Bidi_Class: AL AL AL B L L L
963 assert_eq!(reorder_paras("ابج\nabc"), vec!["\nجبا", "abc"]);
964
965 assert_eq!(reorder_paras("1.-2"), vec!["1.-2"]);
966 assert_eq!(reorder_paras("1-.2"), vec!["1-.2"]);
967 assert_eq!(reorder_paras("abc אבג"), vec!["abc גבא"]);
968
969 // Numbers being weak LTR characters, cannot reorder strong RTL
970 assert_eq!(reorder_paras("123 אבג"), vec!["גבא 123"]);
971
972 assert_eq!(reorder_paras("abc\u{202A}def"), vec!["abc\u{202A}def"]);
973
974 assert_eq!(
975 reorder_paras("abc\u{202A}def\u{202C}ghi"),
976 vec!["abc\u{202A}def\u{202C}ghi"]
977 );
978
979 assert_eq!(
980 reorder_paras("abc\u{2066}def\u{2069}ghi"),
981 vec!["abc\u{2066}def\u{2069}ghi"]
982 );
983
984 // Testing for RLE Character
985 assert_eq!(
986 reorder_paras("\u{202B}abc אבג\u{202C}"),
987 vec!["\u{202B}\u{202C}גבא abc"]
988 );
989
990 // Testing neutral characters
991 assert_eq!(reorder_paras("אבג? אבג"), vec!["גבא ?גבא"]);
992
993 // Testing neutral characters with special case
994 assert_eq!(reorder_paras("A אבג?"), vec!["A גבא?"]);
995
996 // Testing neutral characters with Implicit RTL Marker
997 assert_eq!(reorder_paras("A אבג?\u{200F}"), vec!["A \u{200F}?גבא"]);
998 assert_eq!(reorder_paras("אבג abc"), vec!["abc גבא"]);
999 assert_eq!(
1000 reorder_paras("abc\u{2067}.-\u{2069}ghi"),
1001 vec!["abc\u{2067}-.\u{2069}ghi"]
1002 );
1003
1004 assert_eq!(
1005 reorder_paras("Hello, \u{2068}\u{202E}world\u{202C}\u{2069}!"),
1006 vec!["Hello, \u{2068}\u{202E}\u{202C}dlrow\u{2069}!"]
1007 );
1008
1009 // With mirrorable characters in RTL run
1010 assert_eq!(reorder_paras("א(ב)ג."), vec![".ג)ב(א"]);
1011
1012 // With mirrorable characters on level boundry
1013 assert_eq!(reorder_paras("אב(גד[&ef].)gh"), vec!["gh).]ef&[דג(בא"]);
1014 }
1015
1016 fn reordered_levels_for_paras(text: &str) -> Vec<Vec<Level>> {
1017 let bidi_info = BidiInfo::new(text, None);
1018 bidi_info
1019 .paragraphs
1020 .iter()
1021 .map(|para| bidi_info.reordered_levels(para, para.range.clone()))
1022 .collect()
1023 }
1024
1025 fn reordered_levels_per_char_for_paras(text: &str) -> Vec<Vec<Level>> {
1026 let bidi_info = BidiInfo::new(text, None);
1027 bidi_info
1028 .paragraphs
1029 .iter()
1030 .map(|para| bidi_info.reordered_levels_per_char(para, para.range.clone()))
1031 .collect()
1032 }
1033
1034 #[test]
1035 #[cfg(feature = "hardcoded-data")]
1036 fn test_reordered_levels() {
1037 // BidiTest:946 (LRI PDI)
1038 let text = "\u{2067}\u{2069}";
1039 assert_eq!(
1040 reordered_levels_for_paras(text),
1041 vec![Level::vec(&[0, 0, 0, 0, 0, 0])]
1042 );
1043 assert_eq!(
1044 reordered_levels_per_char_for_paras(text),
1045 vec![Level::vec(&[0, 0])]
1046 );
1047
1048 let text = "aa טֶ";
1049 let bidi_info = BidiInfo::new(text, None);
1050 assert_eq!(
1051 bidi_info.reordered_levels(&bidi_info.paragraphs[0], 3..7),
1052 Level::vec(&[0, 0, 0, 1, 1, 1, 1]),
1053 )
1054
1055 /* TODO
1056 /// BidiTest:69635 (AL ET EN)
1057 let text = "\u{060B}\u{20CF}\u{06F9}";
1058 assert_eq!(
1059 reordered_levels_for_paras(text),
1060 vec![Level::vec(&[1, 1, 1, 1, 1, 2, 2])]
1061 );
1062 assert_eq!(
1063 reordered_levels_per_char_for_paras(text),
1064 vec![Level::vec(&[1, 1, 2])]
1065 );
1066 */
1067
1068 /* TODO
1069 // BidiTest:291284 (AN RLI PDF R)
1070 assert_eq!(
1071 reordered_levels_per_char_for_paras("\u{0605}\u{2067}\u{202C}\u{0590}"),
1072 vec![&["2", "0", "x", "1"]]
1073 );
1074 */
1075 }
1076
1077 #[test]
1078 fn test_paragraph_info_len() {
1079 let text = "hello world";
1080 let bidi_info = BidiInfo::new(text, None);
1081 assert_eq!(bidi_info.paragraphs.len(), 1);
1082 assert_eq!(bidi_info.paragraphs[0].len(), text.len());
1083
1084 let text2 = "How are you";
1085 let whole_text = format!("{}\n{}", text, text2);
1086 let bidi_info = BidiInfo::new(&whole_text, None);
1087 assert_eq!(bidi_info.paragraphs.len(), 2);
1088
1089 // The first paragraph include the paragraph separator.
1090 // TODO: investigate if the paragraph separator character
1091 // should not be part of any paragraph.
1092 assert_eq!(bidi_info.paragraphs[0].len(), text.len() + 1);
1093 assert_eq!(bidi_info.paragraphs[1].len(), text2.len());
1094 }
1095
1096 #[test]
1097 fn test_direction() {
1098 let ltr_text = "hello world";
1099 let rtl_text = "أهلا بكم";
1100 let all_paragraphs = format!("{}\n{}\n{}{}", ltr_text, rtl_text, ltr_text, rtl_text);
1101 let bidi_info = BidiInfo::new(&all_paragraphs, None);
1102 assert_eq!(bidi_info.paragraphs.len(), 3);
1103 let p_ltr = Paragraph::new(&bidi_info, &bidi_info.paragraphs[0]);
1104 let p_rtl = Paragraph::new(&bidi_info, &bidi_info.paragraphs[1]);
1105 let p_mixed = Paragraph::new(&bidi_info, &bidi_info.paragraphs[2]);
1106 assert_eq!(p_ltr.direction(), Direction::Ltr);
1107 assert_eq!(p_rtl.direction(), Direction::Rtl);
1108 assert_eq!(p_mixed.direction(), Direction::Mixed);
1109 }
1110
1111 #[test]
1112 fn test_edge_cases_direction() {
1113 // No paragraphs for empty text.
1114 let empty = "";
1115 let bidi_info = BidiInfo::new(empty, Option::from(RTL_LEVEL));
1116 assert_eq!(bidi_info.paragraphs.len(), 0);
1117 // The paragraph separator will take the value of the default direction
1118 // which is left to right.
1119 let empty = "\n";
1120 let bidi_info = BidiInfo::new(empty, None);
1121 assert_eq!(bidi_info.paragraphs.len(), 1);
1122 let p = Paragraph::new(&bidi_info, &bidi_info.paragraphs[0]);
1123 assert_eq!(p.direction(), Direction::Ltr);
1124 // The paragraph separator will take the value of the given initial direction
1125 // which is left to right.
1126 let empty = "\n";
1127 let bidi_info = BidiInfo::new(empty, Option::from(LTR_LEVEL));
1128 assert_eq!(bidi_info.paragraphs.len(), 1);
1129 let p = Paragraph::new(&bidi_info, &bidi_info.paragraphs[0]);
1130 assert_eq!(p.direction(), Direction::Ltr);
1131
1132 // The paragraph separator will take the value of the given initial direction
1133 // which is right to left.
1134 let empty = "\n";
1135 let bidi_info = BidiInfo::new(empty, Option::from(RTL_LEVEL));
1136 assert_eq!(bidi_info.paragraphs.len(), 1);
1137 let p = Paragraph::new(&bidi_info, &bidi_info.paragraphs[0]);
1138 assert_eq!(p.direction(), Direction::Rtl);
1139 }
1140
1141 #[test]
1142 fn test_level_at() {
1143 let ltr_text = "hello world";
1144 let rtl_text = "أهلا بكم";
1145 let all_paragraphs = format!("{}\n{}\n{}{}", ltr_text, rtl_text, ltr_text, rtl_text);
1146 let bidi_info = BidiInfo::new(&all_paragraphs, None);
1147 assert_eq!(bidi_info.paragraphs.len(), 3);
1148
1149 let p_ltr = Paragraph::new(&bidi_info, &bidi_info.paragraphs[0]);
1150 let p_rtl = Paragraph::new(&bidi_info, &bidi_info.paragraphs[1]);
1151 let p_mixed = Paragraph::new(&bidi_info, &bidi_info.paragraphs[2]);
1152
1153 assert_eq!(p_ltr.level_at(0), LTR_LEVEL);
1154 assert_eq!(p_rtl.level_at(0), RTL_LEVEL);
1155 assert_eq!(p_mixed.level_at(0), LTR_LEVEL);
1156 assert_eq!(p_mixed.info.levels.len(), 54);
1157 assert_eq!(p_mixed.para.range.start, 28);
1158 assert_eq!(p_mixed.level_at(ltr_text.len()), RTL_LEVEL);
1159 }
1160}
1161
1162#[cfg(all(feature = "serde", feature = "hardcoded-data", test))]
1163mod serde_tests {
1164 use super::*;
1165 use serde_test::{assert_tokens, Token};
1166
1167 #[test]
1168 fn test_levels() {
1169 let text = "abc אבג";
1170 let bidi_info = BidiInfo::new(text, None);
1171 let levels = bidi_info.levels;
1172 assert_eq!(text.as_bytes().len(), 10);
1173 assert_eq!(levels.len(), 10);
1174 assert_tokens(
1175 &levels,
1176 &[
1177 Token::Seq { len: Some(10) },
1178 Token::NewtypeStruct { name: "Level" },
1179 Token::U8(0),
1180 Token::NewtypeStruct { name: "Level" },
1181 Token::U8(0),
1182 Token::NewtypeStruct { name: "Level" },
1183 Token::U8(0),
1184 Token::NewtypeStruct { name: "Level" },
1185 Token::U8(0),
1186 Token::NewtypeStruct { name: "Level" },
1187 Token::U8(1),
1188 Token::NewtypeStruct { name: "Level" },
1189 Token::U8(1),
1190 Token::NewtypeStruct { name: "Level" },
1191 Token::U8(1),
1192 Token::NewtypeStruct { name: "Level" },
1193 Token::U8(1),
1194 Token::NewtypeStruct { name: "Level" },
1195 Token::U8(1),
1196 Token::NewtypeStruct { name: "Level" },
1197 Token::U8(1),
1198 Token::SeqEnd,
1199 ],
1200 );
1201 }
1202}
1203