1 | // Copyright (C) 2020 The Qt Company Ltd. |
---|---|
2 | // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only |
3 | |
4 | #include "qunicodetools_p.h" |
5 | |
6 | #include "qunicodetables_p.h" |
7 | #include "qvarlengtharray.h" |
8 | #if QT_CONFIG(library) |
9 | #include "qlibrary.h" |
10 | #endif |
11 | |
12 | #include <limits.h> |
13 | |
14 | #define FLAG(x) (1 << (x)) |
15 | |
16 | QT_BEGIN_NAMESPACE |
17 | |
18 | using namespace Qt::StringLiterals; |
19 | |
20 | #ifdef QT_BUILD_INTERNAL |
21 | Q_CONSTINIT Q_AUTOTEST_EXPORT |
22 | #else |
23 | constexpr |
24 | #endif |
25 | int qt_initcharattributes_default_algorithm_only = 0; |
26 | |
27 | namespace QUnicodeTools { |
28 | |
29 | // ----------------------------------------------------------------------------------------------------- |
30 | // |
31 | // The text boundaries determination algorithm. |
32 | // See https://www.unicode.org/reports/tr29/tr29-37.html |
33 | // |
34 | // ----------------------------------------------------------------------------------------------------- |
35 | |
36 | namespace GB { |
37 | |
38 | // This table is indexed by the grapheme break classes of two |
39 | // (adjacent) code points. |
40 | // The class of the first code point selects an entry. |
41 | // If the entry's bit at position second_cp_class is set |
42 | // (in other words: if entry & (1u << second_cp_class) is non-zero) |
43 | // then there is NO grapheme break between the two code points. |
44 | |
45 | using GBTableEntryType = quint16; |
46 | |
47 | // Check that we have enough bits in the table (in case |
48 | // NumGraphemeBreakClasses grows too much). |
49 | static_assert(sizeof(GBTableEntryType) * CHAR_BIT >= QUnicodeTables::NumGraphemeBreakClasses, |
50 | "Internal error: increase the size in bits of GBTableEntryType"); |
51 | |
52 | // GB9, GB9a |
53 | static const GBTableEntryType Extend_SpacingMark_ZWJ = |
54 | FLAG(QUnicodeTables::GraphemeBreak_Extend) |
55 | | FLAG(QUnicodeTables::GraphemeBreak_SpacingMark) |
56 | | FLAG(QUnicodeTables::GraphemeBreak_ZWJ); |
57 | |
58 | static const GBTableEntryType HardBreak = 0u; |
59 | |
60 | static const GBTableEntryType breakTable[QUnicodeTables::NumGraphemeBreakClasses] = { |
61 | Extend_SpacingMark_ZWJ, // Any |
62 | FLAG(QUnicodeTables::GraphemeBreak_LF), // CR |
63 | HardBreak, // LF |
64 | HardBreak, // Control |
65 | Extend_SpacingMark_ZWJ, // Extend |
66 | Extend_SpacingMark_ZWJ, // ZWJ |
67 | Extend_SpacingMark_ZWJ, // RegionalIndicator |
68 | (Extend_SpacingMark_ZWJ |
69 | | FLAG(QUnicodeTables::GraphemeBreak_Any) |
70 | | FLAG(QUnicodeTables::GraphemeBreak_Prepend) |
71 | | FLAG(QUnicodeTables::GraphemeBreak_L) |
72 | | FLAG(QUnicodeTables::GraphemeBreak_V) |
73 | | FLAG(QUnicodeTables::GraphemeBreak_T) |
74 | | FLAG(QUnicodeTables::GraphemeBreak_LV) |
75 | | FLAG(QUnicodeTables::GraphemeBreak_LVT) |
76 | | FLAG(QUnicodeTables::GraphemeBreak_RegionalIndicator) |
77 | | FLAG(QUnicodeTables::GraphemeBreak_Extended_Pictographic) |
78 | ), // Prepend |
79 | Extend_SpacingMark_ZWJ, // SpacingMark |
80 | (Extend_SpacingMark_ZWJ |
81 | | FLAG(QUnicodeTables::GraphemeBreak_L) |
82 | | FLAG(QUnicodeTables::GraphemeBreak_V) |
83 | | FLAG(QUnicodeTables::GraphemeBreak_LV) |
84 | | FLAG(QUnicodeTables::GraphemeBreak_LVT) |
85 | ), // L |
86 | (Extend_SpacingMark_ZWJ |
87 | | FLAG(QUnicodeTables::GraphemeBreak_V) |
88 | | FLAG(QUnicodeTables::GraphemeBreak_T) |
89 | ), // V |
90 | (Extend_SpacingMark_ZWJ |
91 | | FLAG(QUnicodeTables::GraphemeBreak_T) |
92 | ), // T |
93 | (Extend_SpacingMark_ZWJ |
94 | | FLAG(QUnicodeTables::GraphemeBreak_V) |
95 | | FLAG(QUnicodeTables::GraphemeBreak_T) |
96 | ), // LV |
97 | (Extend_SpacingMark_ZWJ |
98 | | FLAG(QUnicodeTables::GraphemeBreak_T) |
99 | ), // LVT |
100 | Extend_SpacingMark_ZWJ // Extended_Pictographic |
101 | }; |
102 | |
103 | static bool shouldBreakBetweenClasses(QUnicodeTables::GraphemeBreakClass first, |
104 | QUnicodeTables::GraphemeBreakClass second) |
105 | { |
106 | return (breakTable[first] & FLAG(second)) == 0; |
107 | } |
108 | |
109 | // Some rules (GB11, GB12, GB13) cannot be represented by the table alone, |
110 | // so we need to store some local state. |
111 | enum class State : uchar { |
112 | Normal, |
113 | GB11_ExtPicExt, // saw a Extend after a Extended_Pictographic |
114 | GB11_ExtPicExtZWJ, // saw a ZWG after a Extended_Pictographic and zero or more Extend |
115 | GB12_13_RI, // saw a RegionalIndicator following a non-RegionalIndicator |
116 | }; |
117 | |
118 | } // namespace GB |
119 | |
120 | static void getGraphemeBreaks(const char16_t *string, qsizetype len, QCharAttributes *attributes) |
121 | { |
122 | QUnicodeTables::GraphemeBreakClass lcls = QUnicodeTables::GraphemeBreak_LF; // to meet GB1 |
123 | GB::State state = GB::State::Normal; |
124 | for (qsizetype i = 0; i != len; ++i) { |
125 | qsizetype pos = i; |
126 | char32_t ucs4 = string[i]; |
127 | if (QChar::isHighSurrogate(ucs4) && i + 1 != len) { |
128 | ushort low = string[i + 1]; |
129 | if (QChar::isLowSurrogate(ucs4: low)) { |
130 | ucs4 = QChar::surrogateToUcs4(high: ucs4, low); |
131 | ++i; |
132 | } |
133 | } |
134 | |
135 | const QUnicodeTables::Properties *prop = QUnicodeTables::properties(ucs4); |
136 | QUnicodeTables::GraphemeBreakClass cls = (QUnicodeTables::GraphemeBreakClass) prop->graphemeBreakClass; |
137 | |
138 | bool shouldBreak = GB::shouldBreakBetweenClasses(first: lcls, second: cls); |
139 | bool handled = false; |
140 | |
141 | switch (state) { |
142 | case GB::State::Normal: |
143 | break; // will deal with it below |
144 | |
145 | case GB::State::GB11_ExtPicExt: |
146 | Q_ASSERT(lcls == QUnicodeTables::GraphemeBreak_Extend); |
147 | if (cls == QUnicodeTables::GraphemeBreak_Extend) { |
148 | // keep going in the current state |
149 | Q_ASSERT(!shouldBreak); // GB9, do not break before Extend |
150 | handled = true; |
151 | } else if (cls == QUnicodeTables::GraphemeBreak_ZWJ) { |
152 | state = GB::State::GB11_ExtPicExtZWJ; |
153 | Q_ASSERT(!shouldBreak); // GB9, do not break before ZWJ |
154 | handled = true; |
155 | } else { |
156 | state = GB::State::Normal; |
157 | } |
158 | break; |
159 | |
160 | case GB::State::GB11_ExtPicExtZWJ: |
161 | Q_ASSERT(lcls == QUnicodeTables::GraphemeBreak_ZWJ); |
162 | if (cls == QUnicodeTables::GraphemeBreak_Extended_Pictographic) { |
163 | shouldBreak = false; |
164 | handled = true; |
165 | } |
166 | |
167 | state = GB::State::Normal; |
168 | break; |
169 | |
170 | case GB::State::GB12_13_RI: |
171 | Q_ASSERT(lcls == QUnicodeTables::GraphemeBreak_RegionalIndicator); |
172 | if (cls == QUnicodeTables::GraphemeBreak_RegionalIndicator) { |
173 | shouldBreak = false; |
174 | handled = true; |
175 | } |
176 | |
177 | state = GB::State::Normal; |
178 | break; |
179 | } |
180 | |
181 | if (!handled) { |
182 | Q_ASSERT(state == GB::State::Normal); |
183 | if (lcls == QUnicodeTables::GraphemeBreak_Extended_Pictographic) { // GB11 |
184 | if (cls == QUnicodeTables::GraphemeBreak_Extend) { |
185 | state = GB::State::GB11_ExtPicExt; |
186 | Q_ASSERT(!shouldBreak); // GB9, do not break before Extend |
187 | } else if (cls == QUnicodeTables::GraphemeBreak_ZWJ) { |
188 | state = GB::State::GB11_ExtPicExtZWJ; |
189 | Q_ASSERT(!shouldBreak); // GB9, do not break before ZWJ |
190 | } |
191 | } else if (cls == QUnicodeTables::GraphemeBreak_RegionalIndicator) { // GB12, GB13 |
192 | state = GB::State::GB12_13_RI; |
193 | } |
194 | } |
195 | |
196 | if (shouldBreak) |
197 | attributes[pos].graphemeBoundary = true; |
198 | |
199 | lcls = cls; |
200 | } |
201 | |
202 | attributes[len].graphemeBoundary = true; // GB2 |
203 | } |
204 | |
205 | |
206 | namespace WB { |
207 | |
208 | enum Action { |
209 | NoBreak, |
210 | Break, |
211 | Lookup, |
212 | LookupW |
213 | }; |
214 | |
215 | static const uchar breakTable[QUnicodeTables::NumWordBreakClasses][QUnicodeTables::NumWordBreakClasses] = { |
216 | // Any CR LF Newline Extend ZWJ Format RI Katakana HLetter ALetter SQuote DQuote MidNumLet MidLetter MidNum Numeric ExtNumLet WSeg |
217 | { Break , Break , Break , Break , NoBreak, NoBreak, NoBreak, Break , Break , Break , Break , Break , Break , Break , Break , Break , Break , Break , Break }, // Any |
218 | { Break , Break , NoBreak, Break , Break , Break , Break , Break , Break , Break , Break , Break , Break , Break , Break , Break , Break , Break , Break }, // CR |
219 | { Break , Break , Break , Break , Break , Break , Break , Break , Break , Break , Break , Break , Break , Break , Break , Break , Break , Break , Break }, // LF |
220 | { Break , Break , Break , Break , Break , Break , Break , Break , Break , Break , Break , Break , Break , Break , Break , Break , Break , Break , Break }, // Newline |
221 | { Break , Break , Break , Break , NoBreak, NoBreak, NoBreak, Break , Break , Break , Break , Break , Break , Break , Break , Break , Break , Break , Break }, // Extend |
222 | { Break , Break , Break , Break , NoBreak, NoBreak, NoBreak, Break , Break , Break , Break , Break , Break , Break , Break , Break , Break , Break , Break }, // ZWJ |
223 | { Break , Break , Break , Break , NoBreak, NoBreak, NoBreak, Break , Break , Break , Break , Break , Break , Break , Break , Break , Break , Break , Break }, // Format |
224 | { Break , Break , Break , Break , NoBreak, NoBreak, NoBreak, NoBreak, Break , Break , Break , Break , Break , Break , Break , Break , Break , Break , Break }, // RegionalIndicator |
225 | { Break , Break , Break , Break , NoBreak, NoBreak, NoBreak, Break , NoBreak, Break , Break , Break , Break , Break , Break , Break , Break , NoBreak, Break }, // Katakana |
226 | { Break , Break , Break , Break , NoBreak, NoBreak, NoBreak, Break , Break , NoBreak, NoBreak, LookupW, Lookup , LookupW, LookupW, Break , NoBreak, NoBreak, Break }, // HebrewLetter |
227 | { Break , Break , Break , Break , NoBreak, NoBreak, NoBreak, Break , Break , NoBreak, NoBreak, LookupW, Break , LookupW, LookupW, Break , NoBreak, NoBreak, Break }, // ALetter |
228 | { Break , Break , Break , Break , NoBreak, NoBreak, NoBreak, Break , Break , Break , Break , Break , Break , Break , Break , Break , Break , Break , Break }, // SingleQuote |
229 | { Break , Break , Break , Break , NoBreak, NoBreak, NoBreak, Break , Break , Break , Break , Break , Break , Break , Break , Break , Break , Break , Break }, // DoubleQuote |
230 | { Break , Break , Break , Break , NoBreak, NoBreak, NoBreak, Break , Break , Break , Break , Break , Break , Break , Break , Break , Break , Break , Break }, // MidNumLet |
231 | { Break , Break , Break , Break , NoBreak, NoBreak, NoBreak, Break , Break , Break , Break , Break , Break , Break , Break , Break , Break , Break , Break }, // MidLetter |
232 | { Break , Break , Break , Break , NoBreak, NoBreak, NoBreak, Break , Break , Break , Break , Break , Break , Break , Break , Break , Break , Break , Break }, // MidNum |
233 | { Break , Break , Break , Break , NoBreak, NoBreak, NoBreak, Break , Break , NoBreak, NoBreak, Lookup , Break , Lookup , Break , Lookup , NoBreak, NoBreak, Break }, // Numeric |
234 | { Break , Break , Break , Break , NoBreak, NoBreak, NoBreak, Break , NoBreak, NoBreak, NoBreak, Break , Break , Break , Break , Break , NoBreak, NoBreak, Break }, // ExtendNumLet |
235 | { Break , Break , Break , Break , NoBreak, NoBreak, NoBreak, Break , Break , Break , Break , Break , Break , Break , Break , Break , Break , Break , NoBreak }, // WSegSpace |
236 | }; |
237 | |
238 | } // namespace WB |
239 | |
240 | static void getWordBreaks(const char16_t *string, qsizetype len, QCharAttributes *attributes) |
241 | { |
242 | enum WordType { |
243 | WordTypeNone, WordTypeAlphaNumeric, WordTypeHiraganaKatakana |
244 | } currentWordType = WordTypeNone; |
245 | |
246 | QUnicodeTables::WordBreakClass cls = QUnicodeTables::WordBreak_LF; // to meet WB1 |
247 | auto real_cls = cls; // Unaffected by WB4 |
248 | |
249 | for (qsizetype i = 0; i != len; ++i) { |
250 | qsizetype pos = i; |
251 | char32_t ucs4 = string[i]; |
252 | if (QChar::isHighSurrogate(ucs4) && i + 1 != len) { |
253 | ushort low = string[i + 1]; |
254 | if (QChar::isLowSurrogate(ucs4: low)) { |
255 | ucs4 = QChar::surrogateToUcs4(high: ucs4, low); |
256 | ++i; |
257 | } |
258 | } |
259 | |
260 | const QUnicodeTables::Properties *prop = QUnicodeTables::properties(ucs4); |
261 | QUnicodeTables::WordBreakClass ncls = (QUnicodeTables::WordBreakClass) prop->wordBreakClass; |
262 | if (qt_initcharattributes_default_algorithm_only) { |
263 | // as of Unicode 5.1, some punctuation marks were mapped to MidLetter and MidNumLet |
264 | // which caused "hi.there" to be treated like if it were just a single word; |
265 | // we keep the pre-5.1 behavior by remapping these characters in the Unicode tables generator |
266 | // and this code is needed to pass the coverage tests; remove once the issue is fixed. |
267 | if (ucs4 == 0x002E) // FULL STOP |
268 | ncls = QUnicodeTables::WordBreak_MidNumLet; |
269 | else if (ucs4 == 0x003A) // COLON |
270 | ncls = QUnicodeTables::WordBreak_MidLetter; |
271 | } |
272 | |
273 | uchar action = WB::breakTable[cls][ncls]; |
274 | switch (action) { |
275 | case WB::Break: |
276 | if (Q_UNLIKELY(real_cls == QUnicodeTables::WordBreak_ZWJ |
277 | && prop->graphemeBreakClass |
278 | == QUnicodeTables::GraphemeBreak_Extended_Pictographic)) { |
279 | // WB3c: ZWJ × \p{Extended_Pictographic} |
280 | action = WB::NoBreak; |
281 | } |
282 | break; |
283 | case WB::NoBreak: |
284 | if (Q_UNLIKELY(ncls == QUnicodeTables::WordBreak_Extend || ncls == QUnicodeTables::WordBreak_ZWJ || ncls == QUnicodeTables::WordBreak_Format)) { |
285 | // WB4: X(Extend|Format)* -> X |
286 | real_cls = ncls; |
287 | continue; |
288 | } |
289 | if (Q_UNLIKELY(cls == QUnicodeTables::WordBreak_RegionalIndicator)) { |
290 | // WB15/WB16: break between pairs of Regional indicator |
291 | ncls = QUnicodeTables::WordBreak_Any; |
292 | } |
293 | if (Q_UNLIKELY(ncls == QUnicodeTables::WordBreak_WSegSpace |
294 | && real_cls != QUnicodeTables::WordBreak_WSegSpace)) { |
295 | // WB3d should not be affected by WB4 |
296 | action = WB::Break; |
297 | } |
298 | break; |
299 | case WB::Lookup: |
300 | case WB::LookupW: |
301 | for (qsizetype lookahead = i + 1; lookahead < len; ++lookahead) { |
302 | ucs4 = string[lookahead]; |
303 | if (QChar::isHighSurrogate(ucs4) && lookahead + 1 != len) { |
304 | ushort low = string[lookahead + 1]; |
305 | if (QChar::isLowSurrogate(ucs4: low)) { |
306 | ucs4 = QChar::surrogateToUcs4(high: ucs4, low); |
307 | ++lookahead; |
308 | } |
309 | } |
310 | |
311 | prop = QUnicodeTables::properties(ucs4); |
312 | QUnicodeTables::WordBreakClass tcls = (QUnicodeTables::WordBreakClass) prop->wordBreakClass; |
313 | |
314 | if (Q_UNLIKELY(tcls == QUnicodeTables::WordBreak_Extend || tcls == QUnicodeTables::WordBreak_ZWJ || tcls == QUnicodeTables::WordBreak_Format)) { |
315 | // WB4: X(Extend|Format)* -> X |
316 | continue; |
317 | } |
318 | |
319 | if (Q_LIKELY(tcls == cls || (action == WB::LookupW && (tcls == QUnicodeTables::WordBreak_HebrewLetter |
320 | || tcls == QUnicodeTables::WordBreak_ALetter)))) { |
321 | i = lookahead; |
322 | ncls = tcls; |
323 | action = WB::NoBreak; |
324 | } |
325 | break; |
326 | } |
327 | if (action != WB::NoBreak) { |
328 | action = WB::Break; |
329 | if (Q_UNLIKELY(ncls == QUnicodeTables::WordBreak_SingleQuote && cls == QUnicodeTables::WordBreak_HebrewLetter)) |
330 | action = WB::NoBreak; // WB7a |
331 | } |
332 | break; |
333 | } |
334 | |
335 | cls = ncls; |
336 | real_cls = ncls; |
337 | |
338 | if (action == WB::Break) { |
339 | attributes[pos].wordBreak = true; |
340 | if (currentWordType != WordTypeNone) |
341 | attributes[pos].wordEnd = true; |
342 | switch (cls) { |
343 | case QUnicodeTables::WordBreak_Katakana: |
344 | currentWordType = WordTypeHiraganaKatakana; |
345 | attributes[pos].wordStart = true; |
346 | break; |
347 | case QUnicodeTables::WordBreak_HebrewLetter: |
348 | case QUnicodeTables::WordBreak_ALetter: |
349 | case QUnicodeTables::WordBreak_Numeric: |
350 | currentWordType = WordTypeAlphaNumeric; |
351 | attributes[pos].wordStart = true; |
352 | break; |
353 | default: |
354 | currentWordType = WordTypeNone; |
355 | break; |
356 | } |
357 | } |
358 | } |
359 | |
360 | if (currentWordType != WordTypeNone) |
361 | attributes[len].wordEnd = true; |
362 | attributes[len].wordBreak = true; // WB2 |
363 | } |
364 | |
365 | |
366 | namespace SB { |
367 | |
368 | enum State { |
369 | Initial, |
370 | Lower, |
371 | Upper, |
372 | LUATerm, |
373 | ATerm, |
374 | ATermC, |
375 | ACS, |
376 | STerm, |
377 | STermC, |
378 | SCS, |
379 | BAfterC, |
380 | BAfter, |
381 | Break, |
382 | Lookup |
383 | }; |
384 | |
385 | static const uchar breakTable[BAfter + 1][QUnicodeTables::NumSentenceBreakClasses] = { |
386 | // Any CR LF Sep Extend Sp Lower Upper OLetter Numeric ATerm SContinue STerm Close |
387 | { Initial, BAfterC, BAfter , BAfter , Initial, Initial, Lower , Upper , Initial, Initial, ATerm , Initial, STerm , Initial }, // Initial |
388 | { Initial, BAfterC, BAfter , BAfter , Lower , Initial, Initial, Initial, Initial, Initial, LUATerm, Initial, STerm , Initial }, // Lower |
389 | { Initial, BAfterC, BAfter , BAfter , Upper , Initial, Initial, Upper , Initial, Initial, LUATerm, Initial, STerm , Initial }, // Upper |
390 | |
391 | { Lookup , BAfterC, BAfter , BAfter , LUATerm, ACS , Initial, Upper , Break , Initial, ATerm , STerm , STerm , ATermC }, // LUATerm |
392 | { Lookup , BAfterC, BAfter , BAfter , ATerm , ACS , Initial, Break , Break , Initial, ATerm , STerm , STerm , ATermC }, // ATerm |
393 | { Lookup , BAfterC, BAfter , BAfter , ATermC , ACS , Initial, Break , Break , Lookup , ATerm , STerm , STerm , ATermC }, // ATermC |
394 | { Lookup , BAfterC, BAfter , BAfter , ACS , ACS , Initial, Break , Break , Lookup , ATerm , STerm , STerm , Lookup }, // ACS |
395 | |
396 | { Break , BAfterC, BAfter , BAfter , STerm , SCS , Break , Break , Break , Break , ATerm , STerm , STerm , STermC }, // STerm, |
397 | { Break , BAfterC, BAfter , BAfter , STermC , SCS , Break , Break , Break , Break , ATerm , STerm , STerm , STermC }, // STermC |
398 | { Break , BAfterC, BAfter , BAfter , SCS , SCS , Break , Break , Break , Break , ATerm , STerm , STerm , Break }, // SCS |
399 | { Break , Break , BAfter , Break , Break , Break , Break , Break , Break , Break , Break , Break , Break , Break }, // BAfterC |
400 | { Break , Break , Break , Break , Break , Break , Break , Break , Break , Break , Break , Break , Break , Break }, // BAfter |
401 | }; |
402 | |
403 | } // namespace SB |
404 | |
405 | static void getSentenceBreaks(const char16_t *string, qsizetype len, QCharAttributes *attributes) |
406 | { |
407 | uchar state = SB::BAfter; // to meet SB1 |
408 | for (qsizetype i = 0; i != len; ++i) { |
409 | qsizetype pos = i; |
410 | char32_t ucs4 = string[i]; |
411 | if (QChar::isHighSurrogate(ucs4) && i + 1 != len) { |
412 | ushort low = string[i + 1]; |
413 | if (QChar::isLowSurrogate(ucs4: low)) { |
414 | ucs4 = QChar::surrogateToUcs4(high: ucs4, low); |
415 | ++i; |
416 | } |
417 | } |
418 | |
419 | const QUnicodeTables::Properties *prop = QUnicodeTables::properties(ucs4); |
420 | QUnicodeTables::SentenceBreakClass ncls = (QUnicodeTables::SentenceBreakClass) prop->sentenceBreakClass; |
421 | |
422 | Q_ASSERT(state <= SB::BAfter); |
423 | state = SB::breakTable[state][ncls]; |
424 | if (Q_UNLIKELY(state == SB::Lookup)) { // SB8 |
425 | state = SB::Break; |
426 | for (qsizetype lookahead = i + 1; lookahead < len; ++lookahead) { |
427 | ucs4 = string[lookahead]; |
428 | if (QChar::isHighSurrogate(ucs4) && lookahead + 1 != len) { |
429 | ushort low = string[lookahead + 1]; |
430 | if (QChar::isLowSurrogate(ucs4: low)) { |
431 | ucs4 = QChar::surrogateToUcs4(high: ucs4, low); |
432 | ++lookahead; |
433 | } |
434 | } |
435 | |
436 | prop = QUnicodeTables::properties(ucs4); |
437 | QUnicodeTables::SentenceBreakClass tcls = (QUnicodeTables::SentenceBreakClass) prop->sentenceBreakClass; |
438 | switch (tcls) { |
439 | case QUnicodeTables::SentenceBreak_Any: |
440 | case QUnicodeTables::SentenceBreak_Extend: |
441 | case QUnicodeTables::SentenceBreak_Sp: |
442 | case QUnicodeTables::SentenceBreak_Numeric: |
443 | case QUnicodeTables::SentenceBreak_SContinue: |
444 | case QUnicodeTables::SentenceBreak_Close: |
445 | continue; |
446 | case QUnicodeTables::SentenceBreak_Lower: |
447 | i = lookahead; |
448 | state = SB::Initial; |
449 | break; |
450 | default: |
451 | break; |
452 | } |
453 | break; |
454 | } |
455 | } |
456 | if (Q_UNLIKELY(state == SB::Break)) { |
457 | attributes[pos].sentenceBoundary = true; |
458 | state = SB::breakTable[SB::Initial][ncls]; |
459 | } |
460 | } |
461 | |
462 | attributes[len].sentenceBoundary = true; // SB2 |
463 | } |
464 | |
465 | |
466 | // ----------------------------------------------------------------------------------------------------- |
467 | // |
468 | // The line breaking algorithm. |
469 | // See http://www.unicode.org/reports/tr14/tr14-39.html |
470 | // |
471 | // ----------------------------------------------------------------------------------------------------- |
472 | |
473 | namespace LB { |
474 | |
475 | namespace NS { // Number Sequence |
476 | |
477 | // This namespace is used to implement LB25 which, as of Unicode 16, has this |
478 | // definition: |
479 | // NU ( SY | IS )* CL × PO |
480 | // NU ( SY | IS )* CP × PO |
481 | // NU ( SY | IS )* CL × PR |
482 | // NU ( SY | IS )* CP × PR |
483 | // NU ( SY | IS )* × PO |
484 | // NU ( SY | IS )* × PR |
485 | // PO × OP NU |
486 | // PO × OP IS NU |
487 | // PO × NU |
488 | // PR × OP NU |
489 | // PR × OP IS NU |
490 | // PR × NU |
491 | // HY × NU |
492 | // IS × NU |
493 | // NU ( SY | IS )* × NU |
494 | |
495 | enum Action { |
496 | None, |
497 | Start, |
498 | Continue, |
499 | Break, |
500 | NeedOPNU, // Like Start, but must be followed by sequence `(OP (IS)?)? NU` |
501 | // These are 'synthetic' actions and are not used in the table but are |
502 | // tracked otherwise in the code for LB25, to track the state of specific |
503 | // sequences: |
504 | CNeedNU, // Like Continue, but must be followed by NU |
505 | CNeedISNU, // Like Continue, but must be followed by IS? NU |
506 | }; |
507 | |
508 | enum Class { |
509 | XX, |
510 | PRPO, |
511 | OP, |
512 | HY, |
513 | NU, |
514 | SY, |
515 | IS, |
516 | CLCP |
517 | }; |
518 | |
519 | static const uchar actionTable[CLCP + 1][CLCP + 1] = { |
520 | // XX PRPO OP HY NU SY IS CLCP |
521 | { None , NeedOPNU, Start , None , Start , None , None , None }, // XX |
522 | { None , NeedOPNU, Continue, Break , Start , None , None , None }, // PRPO |
523 | { None , Start , Start , Break , Continue, None , Continue, None }, // OP |
524 | { None , None , None , Start , Continue, None , None , None }, // HY |
525 | { Break , Break , Break , Break , Continue, Continue, Continue, Continue }, // NU |
526 | { Break , Break , Break , Break , Continue, Continue, Continue, Continue }, // SY |
527 | { Break , Break , Break , Break , Continue, Continue, Continue, Continue }, // IS |
528 | { Break , Continue, Break , Break , Break , Break , Break , Break }, // CLCP |
529 | }; |
530 | |
531 | inline Class toClass(QUnicodeTables::LineBreakClass lbc) |
532 | { |
533 | switch (lbc) { |
534 | case QUnicodeTables::LineBreak_PR: case QUnicodeTables::LineBreak_PO: |
535 | return PRPO; |
536 | case QUnicodeTables::LineBreak_OP: |
537 | return OP; |
538 | case QUnicodeTables::LineBreak_HY: |
539 | return HY; |
540 | case QUnicodeTables::LineBreak_NU: |
541 | return NU; |
542 | case QUnicodeTables::LineBreak_SY: |
543 | return SY; |
544 | case QUnicodeTables::LineBreak_IS: |
545 | return IS; |
546 | case QUnicodeTables::LineBreak_CL: case QUnicodeTables::LineBreak_CP: |
547 | return CLCP; |
548 | default: |
549 | break; |
550 | } |
551 | return XX; |
552 | } |
553 | |
554 | } // namespace NS |
555 | |
556 | namespace BRS { // Brahmic Sequence, used to implement LB28a |
557 | constexpr char32_t DottedCircle = U'\u25CC'; |
558 | |
559 | // The LB28a_{n} value maps to the 'regex' on the nth line in LB28a |
560 | // The only special case is LB28a_2VI which is a direct match to the 2nd |
561 | // line, but it also leads to LB28a_3VIAK, the 3rd line. |
562 | enum State { |
563 | None, |
564 | Start, // => Have: `(AK | [◌] | AS)` |
565 | LB28a_2VF, // => Have: `(AK | [◌] | AS) VF` |
566 | LB28a_2VI, // => Have: `(AK | [◌] | AS) VI` May find: `(AK | [◌])` |
567 | LB28a_3VIAK, // => Have: `(AK | [◌] | AS) VI (AK | [◌])` |
568 | LB28a_4, // => Have: `(AK | [◌] | AS) (AK | [◌] | AS)` May find: `VF` |
569 | LB28a_4VF, // => Have: `(AK | [◌] | AS) (AK | [◌] | AS) VF` |
570 | Restart, |
571 | }; |
572 | struct LinebreakUnit { |
573 | QUnicodeTables::LineBreakClass lbc; |
574 | char32_t ucs4; |
575 | }; |
576 | struct ParseState { |
577 | State state = None; |
578 | qsizetype start = 0; |
579 | }; |
580 | State updateState(State state, LinebreakUnit lb) |
581 | { |
582 | using LBC = QUnicodeTables::LineBreakClass; |
583 | if (lb.lbc == LBC::LineBreak_CM) |
584 | return state; |
585 | |
586 | switch (state) { |
587 | case Start: |
588 | if (lb.lbc == LBC::LineBreak_VF) |
589 | return LB28a_2VF; |
590 | if (lb.lbc == LBC::LineBreak_VI) |
591 | return LB28a_2VI; |
592 | if (lb.ucs4 == DottedCircle || lb.lbc == LBC::LineBreak_AK |
593 | || lb.lbc == LBC::LineBreak_AS) |
594 | return LB28a_4; |
595 | break; |
596 | case LB28a_2VI: |
597 | if (lb.ucs4 == DottedCircle || lb.lbc == LBC::LineBreak_AK) |
598 | return LB28a_3VIAK; |
599 | break; |
600 | case LB28a_4: |
601 | if (lb.lbc == LBC::LineBreak_VF) |
602 | return LB28a_4VF; |
603 | // Had (AK | [◌] | AS) (AK | [◌] | AS), which could mean the 2nd capture is the start |
604 | // of a new sequence, so we need to check if it makes sense. |
605 | return Restart; |
606 | case None: |
607 | if (Q_UNLIKELY(lb.ucs4 == DottedCircle || lb.lbc == LBC::LineBreak_AK |
608 | || lb.lbc == LBC::LineBreak_AS)) { |
609 | return Start; |
610 | } |
611 | break; |
612 | case LB28a_2VF: |
613 | case LB28a_4VF: |
614 | case LB28a_3VIAK: |
615 | case Restart: |
616 | // These are all terminal states, so no need to update |
617 | Q_UNREACHABLE(); |
618 | } |
619 | return None; |
620 | } |
621 | } |
622 | |
623 | enum Action { |
624 | ProhibitedBreak, PB = ProhibitedBreak, |
625 | DirectBreak, DB = DirectBreak, |
626 | IndirectBreak, IB = IndirectBreak, |
627 | CombiningIndirectBreak, CI = CombiningIndirectBreak, |
628 | CombiningProhibitedBreak, CP = CombiningProhibitedBreak, |
629 | ProhibitedBreakAfterHebrewPlusHyphen, HH = ProhibitedBreakAfterHebrewPlusHyphen, |
630 | IndirectBreakIfNarrow, IN = IndirectBreakIfNarrow, // For LB30 |
631 | DirectBreakOutsideNumericSequence, DN = DirectBreakOutsideNumericSequence, // For LB25 |
632 | }; |
633 | |
634 | // See https://www.unicode.org/reports/tr14/tr14-37.html for the information |
635 | // about the table. It was removed in the later versions of the standard. |
636 | static const uchar breakTable[QUnicodeTables::LineBreak_ZWJ][QUnicodeTables::LineBreak_ZWJ] = { |
637 | /* 1↓ 2→ OP CL CP QU +Pi +Pf +19 GL NS EX SY IS PR PO NU AL HL ID IN HY +WS BA +WS HYBA BB B2 ZW CM WJ H2 H3 JL JV JT RI CB EB EM AK AP AS VI VF*/ |
638 | /* OP */ { PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, CP, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB }, |
639 | /* CL */ { DB, PB, PB, IB, IB, PB, IB, IB, PB, PB, PB, PB, DB, DB, DB, DB, DB, DB, IB, IB, IB, IB, IB, IB, DB, DB, PB, CI, PB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB }, |
640 | /* CP */ { DB, PB, PB, IB, IB, PB, IB, IB, PB, PB, PB, PB, DB, DB, IB, IB, IB, DB, IB, IB, IB, IB, IB, IB, DB, DB, PB, CI, PB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB }, |
641 | /* QU */ { IB, PB, PB, IB, IB, PB, IB, IB, IB, PB, PB, PB, IB, IB, IB, IB, IB, DB, IB, IB, IB, IB, IB, IB, IB, IB, PB, CI, PB, IB, IB, IB, IB, IB, IB, IB, IB, IB, IB, IB, IB, IB, IB }, |
642 | /* +Pi*/ { PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, CP, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB }, |
643 | /* +Pf*/ { IB, PB, PB, IB, IB, PB, IB, IB, IB, PB, PB, PB, IB, IB, IB, IB, IB, IB, IB, IB, IB, IB, IB, IB, IB, IB, PB, CI, PB, IB, IB, IB, IB, IB, IB, IB, IB, IB, IB, IB, IB, IB, IB }, |
644 | /* +19*/ { IB, PB, PB, IB, IB, PB, IB, IB, IB, PB, PB, PB, IB, IB, IB, IB, IB, IB, IB, IB, IB, IB, IB, IB, IB, IB, PB, CI, PB, IB, IB, IB, IB, IB, IB, IB, IB, IB, IB, IB, IB, IB, IB }, |
645 | /* GL */ { IB, PB, PB, IB, IB, PB, IB, IB, IB, PB, PB, PB, IB, IB, IB, IB, IB, IB, IB, IB, IB, IB, IB, IB, IB, IB, PB, CI, PB, IB, IB, IB, IB, IB, IB, IB, IB, IB, IB, IB, IB, IB, IB }, |
646 | /* NS */ { DB, PB, PB, DB, IB, PB, IB, IB, IB, PB, PB, PB, DB, DB, DB, DB, DB, DB, IB, IB, IB, IB, IB, IB, DB, DB, PB, CI, PB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB }, |
647 | /* EX */ { DB, PB, PB, IB, IB, PB, IB, IB, IB, PB, PB, PB, DB, DB, DB, DB, DB, DB, IB, IB, IB, IB, IB, IB, DB, DB, PB, CI, PB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB }, |
648 | /* SY */ { DB, PB, PB, IB, IB, PB, IB, IB, IB, PB, PB, PB, DB, DB, DB, DB, IB, DB, IB, IB, IB, IB, IB, IB, DB, DB, PB, CI, PB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB }, |
649 | /* IS */ { DB, PB, PB, IB, IB, PB, IB, IB, IB, PB, PB, PB, DN, DB, IB, IB, IB, DB, IB, IB, IB, IB, IB, IB, DB, DB, PB, CI, PB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB }, |
650 | /* PR */ { DB, PB, PB, IB, IB, PB, IB, IB, IB, PB, PB, PB, DB, DB, IB, IB, IB, IB, IB, IB, IB, IB, IB, IB, DB, DB, PB, CI, PB, IB, IB, IB, IB, IB, DB, DB, IB, IB, DB, DB, DB, DB, DB }, |
651 | /* PO */ { DB, PB, PB, IB, IB, PB, IB, IB, IB, PB, PB, PB, DB, DB, IB, IB, IB, DB, IB, IB, IB, IB, IB, IB, DB, DB, PB, CI, PB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB }, |
652 | /* NU */ { IN, PB, PB, IB, IB, PB, IB, IB, IB, PB, PB, PB, IB, IB, IB, IB, IB, DB, IB, IB, IB, IB, IB, IB, DB, DB, PB, CI, PB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB }, |
653 | /* AL */ { IN, PB, PB, IB, IB, PB, IB, IB, IB, PB, PB, PB, IB, IB, IB, IB, IB, DB, IB, IB, IB, IB, IB, IB, DB, DB, PB, CI, PB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB }, |
654 | /* HL */ { IN, PB, PB, IB, IB, PB, IB, IB, IB, PB, PB, PB, IB, IB, IB, IB, IB, DB, IB, IB, CI, CI, CI, CI, DB, DB, PB, CI, PB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB }, |
655 | /* ID */ { DB, PB, PB, DB, IB, PB, IB, IB, IB, PB, PB, PB, DB, IB, DB, DB, DB, DB, IB, IB, IB, IB, IB, IB, DB, DB, PB, CI, PB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB }, |
656 | /* IN */ { DB, PB, PB, IB, IB, PB, IB, IB, IB, PB, PB, PB, DB, DB, DB, DB, DB, DB, IB, IB, IB, IB, IB, IB, DB, DB, PB, CI, PB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB }, |
657 | /* HY */ { HH, PB, PB, IB, IB, PB, IB, HH, IB, PB, PB, PB, HH, HH, IB, HH, HH, HH, IB, IB, IB, IB, IB, IB, HH, HH, PB, CI, PB, HH, HH, HH, HH, HH, HH, DB, DB, DB, HH, HH, HH, HH, HH }, |
658 | /* +WS*/ { HH, PB, PB, IB, IB, PB, IB, HH, IB, PB, PB, PB, HH, HH, IB, IB, HH, HH, IB, IB, IB, IB, IB, IB, HH, HH, PB, CI, PB, HH, HH, HH, HH, HH, HH, DB, DB, DB, HH, HH, HH, HH, HH }, |
659 | /* BA */ { HH, PB, PB, IB, IB, PB, IB, HH, IB, PB, PB, PB, HH, HH, HH, HH, HH, HH, IB, IB, IB, IB, IB, IB, HH, HH, PB, CI, PB, HH, HH, HH, HH, HH, HH, DB, DB, DB, HH, HH, HH, HH, HH }, |
660 | /* +WS*/ { HH, PB, PB, IB, IB, PB, IB, HH, IB, PB, PB, PB, HH, HH, HH, IB, HH, HH, IB, IB, IB, IB, IB, IB, HH, HH, PB, CI, PB, HH, HH, HH, HH, HH, HH, DB, DB, DB, HH, HH, HH, HH, HH }, |
661 | /*HYBA*/ { PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, DB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB, PB }, |
662 | /* BB */ { IB, PB, PB, IB, IB, PB, IB, IB, IB, PB, PB, PB, IB, IB, IB, IB, IB, IB, IB, IB, IB, IB, IB, IB, IB, IB, PB, CI, PB, IB, IB, IB, IB, IB, IB, DB, IB, IB, IB, IB, IB, IB, IB }, |
663 | /* B2 */ { DB, PB, PB, IB, IB, PB, IB, IB, IB, PB, PB, PB, DB, DB, DB, DB, DB, DB, IB, IB, IB, IB, IB, IB, DB, PB, PB, CI, PB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB }, |
664 | /* ZW */ { DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, PB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB }, |
665 | /* CM */ { IB, PB, PB, IB, IB, PB, IB, IB, IB, PB, PB, PB, DB, DB, IB, IB, IB, DB, IB, IB, IB, IB, IB, IB, DB, DB, PB, CI, PB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB }, |
666 | /* WJ */ { IB, PB, PB, IB, IB, PB, IB, IB, IB, PB, PB, PB, IB, IB, IB, IB, IB, IB, IB, IB, IB, IB, IB, IB, IB, IB, PB, CI, PB, IB, IB, IB, IB, IB, IB, IB, IB, IB, IB, IB, IB, IB, IB }, |
667 | /* H2 */ { DB, PB, PB, IB, IB, PB, IB, IB, IB, PB, PB, PB, DB, IB, DB, DB, DB, DB, IB, IB, IB, IB, IB, IB, DB, DB, PB, CI, PB, DB, DB, DB, IB, IB, DB, DB, DB, DB, DB, DB, DB, DB, DB }, |
668 | /* H3 */ { DB, PB, PB, IB, IB, PB, IB, IB, IB, PB, PB, PB, DB, IB, DB, DB, DB, DB, IB, IB, IB, IB, IB, IB, DB, DB, PB, CI, PB, DB, DB, DB, DB, IB, DB, DB, DB, DB, DB, DB, DB, DB, DB }, |
669 | /* JL */ { DB, PB, PB, IB, IB, PB, IB, IB, IB, PB, PB, PB, DB, IB, DB, DB, DB, DB, IB, IB, IB, IB, IB, IB, DB, DB, PB, CI, PB, IB, IB, IB, IB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB }, |
670 | /* JV */ { DB, PB, PB, IB, IB, PB, IB, IB, IB, PB, PB, PB, DB, IB, DB, DB, DB, DB, IB, IB, IB, IB, IB, IB, DB, DB, PB, CI, PB, DB, DB, DB, IB, IB, DB, DB, DB, DB, DB, DB, DB, DB, DB }, |
671 | /* JT */ { DB, PB, PB, IB, IB, PB, IB, IB, IB, PB, PB, PB, DB, IB, DB, DB, DB, DB, IB, IB, IB, IB, IB, IB, DB, DB, PB, CI, PB, DB, DB, DB, DB, IB, DB, DB, DB, DB, DB, DB, DB, DB, DB }, |
672 | /* RI */ { DB, PB, PB, IB, IB, PB, IB, IB, IB, PB, PB, PB, DB, DB, DB, DB, DB, DB, IB, IB, IB, IB, IB, IB, DB, DB, PB, CI, PB, DB, DB, DB, DB, DB, IB, DB, DB, DB, DB, DB, DB, DB, DB }, |
673 | /* CB */ { DB, PB, PB, IB, IB, PB, IB, IB, DB, PB, PB, PB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, PB, CI, PB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB }, |
674 | /* EB */ { DB, PB, PB, IB, IB, PB, IB, IB, IB, PB, PB, PB, DB, IB, DB, DB, DB, DB, IB, IB, IB, IB, IB, IB, DB, DB, PB, CI, PB, DB, DB, DB, DB, DB, DB, DB, DB, IB, DB, DB, DB, DB, DB }, |
675 | /* EM */ { DB, PB, PB, IB, IB, PB, IB, IB, IB, PB, PB, PB, DB, IB, DB, DB, DB, DB, IB, IB, IB, IB, IB, IB, DB, DB, PB, CI, PB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB }, |
676 | /* AK */ { DB, PB, PB, IB, IB, PB, IB, IB, IB, PB, PB, PB, DB, DB, DB, DB, DB, DB, IB, IB, IB, IB, IB, IB, DB, DB, PB, DB, PB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, IB, IB }, |
677 | /* AP */ { DB, PB, PB, IB, IB, PB, IB, IB, IB, PB, PB, PB, DB, DB, DB, DB, DB, DB, IB, IB, IB, IB, IB, IB, DB, DB, PB, DB, PB, DB, DB, DB, DB, DB, DB, DB, DB, DB, IB, DB, IB, DB, DB }, |
678 | /* AS */ { DB, PB, PB, IB, IB, PB, IB, IB, IB, PB, PB, PB, DB, DB, DB, DB, DB, DB, IB, IB, IB, IB, IB, IB, DB, DB, PB, DB, PB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, IB, IB }, |
679 | /* VI */ { DB, PB, PB, IB, IB, PB, IB, IB, IB, PB, PB, PB, DB, DB, DB, DB, DB, DB, IB, IB, IB, IB, IB, IB, DB, DB, PB, DB, PB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB }, |
680 | /* VF */ { DB, PB, PB, IB, IB, PB, IB, IB, IB, PB, PB, PB, DB, DB, DB, DB, DB, DB, IB, IB, IB, IB, IB, IB, DB, DB, PB, DB, PB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB, DB }, |
681 | }; |
682 | |
683 | // The following line break classes are not treated by the pair table |
684 | // and must be resolved outside: |
685 | // AI, BK, CB, CJ, CR, LF, NL, SA, SG, SP, XX, ZWJ |
686 | |
687 | } // namespace LB |
688 | |
689 | static void getLineBreaks(const char16_t *string, qsizetype len, QCharAttributes *attributes, QUnicodeTools::CharAttributeOptions options) |
690 | { |
691 | qsizetype nestart = 0; |
692 | LB::NS::Class nelast = LB::NS::XX; |
693 | LB::NS::Action neactlast = LB::NS::None; |
694 | |
695 | LB::BRS::ParseState brsState; |
696 | |
697 | QUnicodeTables::LineBreakClass lcls = QUnicodeTables::LineBreak_LF; // to meet LB10 |
698 | QUnicodeTables::LineBreakClass cls = lcls; |
699 | const QUnicodeTables::Properties *lastProp = QUnicodeTables::properties(ucs4: U'\n'); |
700 | |
701 | constexpr static auto isEastAsian = [](QUnicodeTables::EastAsianWidth eaw) { |
702 | using EAW = QUnicodeTables::EastAsianWidth; |
703 | return eaw == EAW::W || eaw == EAW::F || eaw == EAW::H; |
704 | }; |
705 | |
706 | for (qsizetype i = 0; i != len; ++i) { |
707 | qsizetype pos = i; |
708 | char32_t ucs4 = string[i]; |
709 | if (QChar::isHighSurrogate(ucs4) && i + 1 != len) { |
710 | ushort low = string[i + 1]; |
711 | if (QChar::isLowSurrogate(ucs4: low)) { |
712 | ucs4 = QChar::surrogateToUcs4(high: ucs4, low); |
713 | ++i; |
714 | } |
715 | } |
716 | |
717 | const QUnicodeTables::Properties *prop = QUnicodeTables::properties(ucs4); |
718 | QUnicodeTables::LineBreakClass ncls = (QUnicodeTables::LineBreakClass) prop->lineBreakClass; |
719 | QUnicodeTables::LineBreakClass tcls; |
720 | |
721 | if (options & QUnicodeTools::HangulLineBreakTailoring) { |
722 | if (Q_UNLIKELY((ncls >= QUnicodeTables::LineBreak_H2 |
723 | && ncls <= QUnicodeTables::LineBreak_JT) |
724 | || (ucs4 >= 0x3130 && ucs4 <= 0x318F && ncls == QUnicodeTables::LineBreak_ID)) |
725 | ) { |
726 | // LB27: use SPACE for line breaking |
727 | // "When Korean uses SPACE for line breaking, the classes in rule LB26, |
728 | // as well as characters of class ID, are often tailored to AL; see Section 8, Customization." |
729 | // In case of Korean syllables: "3130..318F HANGUL COMPATIBILITY JAMO" |
730 | ncls = QUnicodeTables::LineBreak_AL; |
731 | } else { |
732 | if (Q_UNLIKELY(ncls == QUnicodeTables::LineBreak_SA)) { |
733 | // LB1: resolve SA to AL, except of those that have Category Mn or Mc be resolved to CM |
734 | static const int test = FLAG(QChar::Mark_NonSpacing) | FLAG(QChar::Mark_SpacingCombining); |
735 | if (FLAG(prop->category) & test) |
736 | ncls = QUnicodeTables::LineBreak_CM; |
737 | } |
738 | } |
739 | } |
740 | |
741 | if (Q_UNLIKELY(ncls == QUnicodeTables::LineBreak_SA)) { |
742 | // LB1: resolve SA to AL, except of those that have Category Mn or Mc be resolved to CM |
743 | static const int test = FLAG(QChar::Mark_NonSpacing) | FLAG(QChar::Mark_SpacingCombining); |
744 | if (FLAG(prop->category) & test) |
745 | ncls = QUnicodeTables::LineBreak_CM; |
746 | } |
747 | |
748 | if (Q_UNLIKELY(ncls == QUnicodeTables::LineBreak_QU)) { |
749 | if (prop->category == QChar::Punctuation_InitialQuote) { |
750 | // LB15a: Do not break after an unresolved initial punctuation |
751 | // that lies at the start of the line, after a space, after |
752 | // opening punctuation, or after an unresolved quotation mark, |
753 | // even after spaces. |
754 | // (sot | BK | CR | LF | NL | OP | QU | GL | SP | ZW) |
755 | // [\p{Pi}&QU] SP* × |
756 | // Note: sot is treated as LF here due to initial loop setup. |
757 | constexpr QUnicodeTables::LineBreakClass lb15a[] = { |
758 | QUnicodeTables::LineBreak_BK, QUnicodeTables::LineBreak_CR, |
759 | QUnicodeTables::LineBreak_LF, QUnicodeTables::LineBreak_OP, |
760 | QUnicodeTables::LineBreak_QU, QUnicodeTables::LineBreak_QU_Pi, |
761 | QUnicodeTables::LineBreak_QU_Pf, QUnicodeTables::LineBreak_GL, |
762 | QUnicodeTables::LineBreak_SP, QUnicodeTables::LineBreak_ZW}; |
763 | if (std::any_of(first: std::begin(arr: lb15a), last: std::end(arr: lb15a), |
764 | pred: [lcls](auto x) { return x == lcls; })) { |
765 | ncls = QUnicodeTables::LineBreak_QU_Pi; |
766 | } |
767 | } else if (prop->category == QChar::Punctuation_FinalQuote) { |
768 | // LB15b: Do not break before an unresolved final punctuation |
769 | // that lies at the end of the line, before a space, before |
770 | // a prohibited break, or before an unresolved quotation mark, |
771 | // even after spaces. |
772 | // × [\p{Pf}&QU] ( SP | GL | WJ | CL | QU | CP | EX | IS |
773 | // | SY | BK | CR | LF | NL | ZW | eot) |
774 | auto nncls = QUnicodeTables::LineBreak_LF; |
775 | |
776 | if (i + 1 < len) { |
777 | char32_t c = string[i + 1]; |
778 | if (QChar::isHighSurrogate(ucs4: c) && i + 2 < len) { |
779 | ushort low = string[i + 2]; |
780 | if (QChar::isLowSurrogate(ucs4: low)) |
781 | c = QChar::surrogateToUcs4(high: c, low); |
782 | } |
783 | nncls = QUnicodeTables::LineBreakClass( |
784 | QUnicodeTables::properties(ucs4: c)->lineBreakClass); |
785 | } |
786 | |
787 | constexpr QUnicodeTables::LineBreakClass lb15b[] = { |
788 | QUnicodeTables::LineBreak_SP, QUnicodeTables::LineBreak_GL, |
789 | QUnicodeTables::LineBreak_WJ, QUnicodeTables::LineBreak_CL, |
790 | QUnicodeTables::LineBreak_QU, QUnicodeTables::LineBreak_QU_Pi, |
791 | QUnicodeTables::LineBreak_QU_Pf, QUnicodeTables::LineBreak_CP, |
792 | QUnicodeTables::LineBreak_EX, QUnicodeTables::LineBreak_IS, |
793 | QUnicodeTables::LineBreak_SY, QUnicodeTables::LineBreak_BK, |
794 | QUnicodeTables::LineBreak_CR, QUnicodeTables::LineBreak_LF, |
795 | QUnicodeTables::LineBreak_ZW}; |
796 | if (std::any_of(first: std::begin(arr: lb15b), last: std::end(arr: lb15b), |
797 | pred: [nncls](auto x) { return x == nncls; })) { |
798 | ncls = QUnicodeTables::LineBreak_QU_Pf; |
799 | } |
800 | } |
801 | } |
802 | |
803 | if (Q_UNLIKELY((lcls >= QUnicodeTables::LineBreak_SP || lcls == QUnicodeTables::LineBreak_ZW |
804 | || lcls == QUnicodeTables::LineBreak_GL |
805 | || lcls == QUnicodeTables::LineBreak_CB) |
806 | && (ncls == QUnicodeTables::LineBreak_HY || ucs4 == u'\u2010'))) { |
807 | // LB20a: Do not break after a word-initial hyphen. |
808 | // ( sot | BK | CR | LF | NL | SP | ZW | CB | GL ) ( HY | [\u2010] ) × AL |
809 | |
810 | // Remap to the synthetic class WS_* (whitespace+*), which is just |
811 | // like the current respective linebreak class but with an IB action |
812 | // if the next class is AL. |
813 | if (ucs4 == u'\u2010') |
814 | ncls = QUnicodeTables::LineBreak_WS_BA; |
815 | else |
816 | ncls = QUnicodeTables::LineBreak_WS_HY; |
817 | } |
818 | |
819 | if (Q_UNLIKELY(cls == QUnicodeTables::LineBreak_AP && ucs4 == LB::BRS::DottedCircle)) { |
820 | // LB28a: Do not break inside the orthographic syllables of Brahmic scripts |
821 | // AP × (AK | [◌] | AS) |
822 | // @note: AP × (AK | AS) is checked by the breakTable |
823 | goto next; |
824 | } |
825 | while (true) { // May need to recheck once. |
826 | // LB28a cont'd |
827 | LB::BRS::State oldState = brsState.state; |
828 | brsState.state = LB::BRS::updateState(state: brsState.state, lb: {.lbc: ncls, .ucs4: ucs4}); |
829 | if (Q_LIKELY(brsState.state == oldState)) |
830 | break; |
831 | switch (brsState.state) { |
832 | case LB::BRS::Start: |
833 | brsState.start = i; |
834 | break; |
835 | case LB::BRS::LB28a_2VI: // Wait for more characters, but also valid sequence |
836 | // We may get another character, but this is already a complete |
837 | // sequence that should not have any breaks: |
838 | for (qsizetype j = brsState.start + 1; j < i; ++j) |
839 | attributes[j].lineBreak = false; |
840 | // No need to mark this sequence again later, so move 'start' |
841 | // up to the current position: |
842 | brsState.start = i; |
843 | goto next; |
844 | case LB::BRS::Restart: |
845 | // The previous character was possibly the start of a new sequence |
846 | brsState.state = LB::BRS::Start; |
847 | brsState.start = pos - 1; |
848 | continue; // Doing the loop again! |
849 | case LB::BRS::LB28a_2VF: |
850 | case LB::BRS::LB28a_4VF: |
851 | case LB::BRS::LB28a_3VIAK: |
852 | for (qsizetype j = brsState.start + 1; j < i; ++j) |
853 | attributes[j].lineBreak = false; |
854 | if (brsState.state == LB::BRS::LB28a_3VIAK) { |
855 | // This might be the start of a new sequence |
856 | brsState.state = LB::BRS::Start; |
857 | brsState.start = i; |
858 | } else { |
859 | brsState.state = LB::BRS::None; |
860 | } |
861 | goto next; |
862 | case LB::BRS::LB28a_4: // Wait for more characters |
863 | case LB::BRS::None: // Nothing to do |
864 | break; |
865 | } |
866 | break; |
867 | } |
868 | |
869 | if (Q_UNLIKELY(ncls == QUnicodeTables::LineBreak_IS)) { |
870 | // LB15c Break before a decimal mark that follows a space, for instance, in |
871 | // ‘subtract .5’. |
872 | if (Q_UNLIKELY(lcls == QUnicodeTables::LineBreak_SP)) { |
873 | if (i + 1 < len) { |
874 | char32_t ch = string[i + 1]; |
875 | if (QChar::isHighSurrogate(ucs4: ch) && i + 2 < len) { |
876 | ushort low = string[i + 2]; |
877 | if (QChar::isLowSurrogate(ucs4: low)) |
878 | ch = QChar::surrogateToUcs4(high: ch, low); |
879 | } |
880 | if (QUnicodeTables::properties(ucs4: ch)->lineBreakClass |
881 | == QUnicodeTables::LineBreak_NU) { |
882 | attributes[pos].lineBreak = true; |
883 | goto next; |
884 | } |
885 | } |
886 | } |
887 | } |
888 | |
889 | if (Q_UNLIKELY(lcls == QUnicodeTables::LineBreak_HL)) { |
890 | // LB21a Do not break after the hyphen in Hebrew + Hyphen + non-Hebrew |
891 | // HL (HY | [ BA - $EastAsian ]) × [^HL] |
892 | auto eaw = QUnicodeTables::EastAsianWidth(prop->eastAsianWidth); |
893 | const bool isNonEaBA = ncls == QUnicodeTables::LineBreak_BA && !isEastAsian(eaw); |
894 | if (isNonEaBA || ncls == QUnicodeTables::LineBreak_HY) { |
895 | // Remap to synthetic HYBA class which handles the next |
896 | // character. Generally (LB21) there are no breaks before |
897 | // HY or BA, so we can skip ahead to the next character. |
898 | ncls = QUnicodeTables::LineBreak_HYBA; |
899 | goto next; |
900 | } |
901 | } |
902 | |
903 | // LB25: do not break lines inside numbers |
904 | { |
905 | LB::NS::Class necur = LB::NS::toClass(lbc: ncls); |
906 | LB::NS::Action neact = LB::NS::Action(LB::NS::actionTable[nelast][necur]); |
907 | if (Q_UNLIKELY(neactlast == LB::NS::CNeedNU && necur != LB::NS::NU)) { |
908 | neact = LB::NS::None; |
909 | } else if (Q_UNLIKELY(neactlast == LB::NS::NeedOPNU)) { |
910 | if (necur == LB::NS::OP) |
911 | neact = LB::NS::CNeedISNU; |
912 | else if (necur == LB::NS::NU) |
913 | neact = LB::NS::Continue; |
914 | else // Anything else and we ignore the sequence |
915 | neact = LB::NS::None; |
916 | } else if (Q_UNLIKELY(neactlast == LB::NS::CNeedISNU)) { |
917 | if (necur == LB::NS::IS) |
918 | neact = LB::NS::CNeedNU; |
919 | else if (necur == LB::NS::NU) |
920 | neact = LB::NS::Continue; |
921 | else // Anything else and we ignore the sequence |
922 | neact = LB::NS::None; |
923 | } |
924 | switch (neact) { |
925 | case LB::NS::Break: |
926 | // do not change breaks before and after the expression |
927 | for (qsizetype j = nestart + 1; j < pos; ++j) |
928 | attributes[j].lineBreak = false; |
929 | Q_FALLTHROUGH(); |
930 | case LB::NS::None: |
931 | nelast = LB::NS::XX; // reset state |
932 | break; |
933 | case LB::NS::NeedOPNU: |
934 | case LB::NS::Start: |
935 | if (neactlast == LB::NS::Start || neactlast == LB::NS::Continue) { |
936 | // Apply the linebreaks for the previous stretch; we need to start a new one |
937 | for (qsizetype j = nestart + 1; j < pos; ++j) |
938 | attributes[j].lineBreak = false; |
939 | } |
940 | nestart = i; |
941 | Q_FALLTHROUGH(); |
942 | case LB::NS::CNeedNU: |
943 | case LB::NS::CNeedISNU: |
944 | case LB::NS::Continue: |
945 | nelast = necur; |
946 | break; |
947 | } |
948 | neactlast = neact; |
949 | } |
950 | |
951 | // LB19a Unless surrounded by East Asian characters, do not break either side of any |
952 | // unresolved quotation marks |
953 | if (Q_UNLIKELY(ncls == QUnicodeTables::LineBreak_QU |
954 | && lcls != QUnicodeTables::LineBreak_SP |
955 | && lcls != QUnicodeTables::LineBreak_ZW)) { |
956 | using EAW = QUnicodeTables::EastAsianWidth; |
957 | constexpr static auto nextCharNonEastAsian = [](const char16_t *string, qsizetype len) { |
958 | if (len > 0) { |
959 | char32_t nch = string[0]; |
960 | if (QChar::isHighSurrogate(ucs4: nch) && len > 1) { |
961 | char16_t low = string[1]; |
962 | if (QChar::isLowSurrogate(ucs4: low)) |
963 | nch = QChar::surrogateToUcs4(high: char16_t(nch), low); |
964 | } |
965 | const auto *nextProp = QUnicodeTables::properties(ucs4: nch); |
966 | QUnicodeTables::LineBreakClass nncls = QUnicodeTables::LineBreakClass( |
967 | nextProp->lineBreakClass); |
968 | QUnicodeTables::EastAsianWidth neaw = EAW(nextProp->eastAsianWidth); |
969 | return nncls != QUnicodeTables::LineBreak_CM |
970 | && nncls <= QUnicodeTables::LineBreak_SP |
971 | && !isEastAsian(neaw); |
972 | } |
973 | return true; // end-of-text counts as non-East-Asian |
974 | }; |
975 | if (Q_UNLIKELY(!isEastAsian(EAW(lastProp->eastAsianWidth)) |
976 | || nextCharNonEastAsian(string + i + 1, len - i - 1))) { |
977 | // Remap to the synthetic QU_19 class which has indirect breaks |
978 | // for most following classes. |
979 | ncls = QUnicodeTables::LineBreak_QU_19; |
980 | } |
981 | } |
982 | |
983 | if (Q_UNLIKELY(lcls >= QUnicodeTables::LineBreak_CR)) { |
984 | // LB4: BK!, LB5: (CRxLF|CR|LF|NL)! |
985 | if (lcls > QUnicodeTables::LineBreak_CR || ncls != QUnicodeTables::LineBreak_LF) |
986 | attributes[pos].lineBreak = attributes[pos].mandatoryBreak = true; |
987 | goto next; |
988 | } |
989 | |
990 | if (Q_UNLIKELY(ncls >= QUnicodeTables::LineBreak_SP)) { |
991 | if (ncls > QUnicodeTables::LineBreak_SP) |
992 | goto next; // LB6: x(BK|CR|LF|NL) |
993 | goto next_no_cls_update; // LB7: xSP |
994 | } |
995 | |
996 | // LB19 - do not break before non-initial unresolved quotation marks, or after non-final |
997 | // unresolved quotation marks |
998 | if (Q_UNLIKELY(((ncls == QUnicodeTables::LineBreak_QU |
999 | || ncls == QUnicodeTables::LineBreak_QU_19) |
1000 | && prop->category != QChar::Punctuation_InitialQuote) |
1001 | || (cls == QUnicodeTables::LineBreak_QU |
1002 | && lastProp->category != QChar::Punctuation_FinalQuote))) { |
1003 | // Make sure the previous character is not one that we have to break after. |
1004 | // Also skip if ncls is CM so it can be treated as lcls (LB9) |
1005 | if (lcls != QUnicodeTables::LineBreak_SP && lcls != QUnicodeTables::LineBreak_ZW |
1006 | && ncls != QUnicodeTables::LineBreak_CM) { |
1007 | goto next; |
1008 | } |
1009 | } |
1010 | |
1011 | if (Q_UNLIKELY(ncls == QUnicodeTables::LineBreak_CM || ncls == QUnicodeTables::LineBreak_ZWJ)) { |
1012 | // LB9: treat CM that don't follows SP, BK, CR, LF, NL, or ZW as X |
1013 | if (lcls != QUnicodeTables::LineBreak_ZW && lcls < QUnicodeTables::LineBreak_SP) |
1014 | // don't update anything |
1015 | goto next_no_cls_update; |
1016 | } |
1017 | |
1018 | if (Q_UNLIKELY(lcls == QUnicodeTables::LineBreak_ZWJ)) { |
1019 | // LB8a: ZWJ x |
1020 | goto next; |
1021 | } |
1022 | |
1023 | if (Q_UNLIKELY(ncls == QUnicodeTables::LineBreak_RI && lcls == QUnicodeTables::LineBreak_RI)) { |
1024 | // LB30a |
1025 | ncls = QUnicodeTables::LineBreak_SP; |
1026 | goto next; |
1027 | } |
1028 | |
1029 | if (Q_UNLIKELY(ncls == QUnicodeTables::LineBreak_EM |
1030 | && lastProp->category == QChar::Other_NotAssigned |
1031 | && lastProp->graphemeBreakClass |
1032 | == QUnicodeTables::GraphemeBreak_Extended_Pictographic)) { |
1033 | // LB30b: [\p{Extended_Pictographic}&\p{Cn}] × EM |
1034 | goto next; |
1035 | } |
1036 | |
1037 | // for South East Asian chars that require a complex analysis, the Unicode |
1038 | // standard recommends to treat them as AL. tailoring that do dictionary analysis can override |
1039 | if (Q_UNLIKELY(cls >= QUnicodeTables::LineBreak_SA)) |
1040 | cls = QUnicodeTables::LineBreak_AL; |
1041 | |
1042 | tcls = cls; |
1043 | |
1044 | constexpr static auto remapToAL = [](QUnicodeTables::LineBreakClass &c, auto &property) { |
1045 | if (Q_UNLIKELY(c == QUnicodeTables::LineBreak_CM |
1046 | || c == QUnicodeTables::LineBreak_ZWJ)) { |
1047 | c = QUnicodeTables::LineBreak_AL; |
1048 | property = QUnicodeTables::properties(ucs4: U'\u0041'); |
1049 | } |
1050 | }; |
1051 | // LB10 Treat any remaining combining mark or ZWJ as AL, |
1052 | // as if it had the properties of U+0041 A LATIN CAPITAL LETTER |
1053 | remapToAL(tcls, lastProp); |
1054 | remapToAL(ncls, prop); |
1055 | |
1056 | switch (LB::breakTable[tcls][ncls < QUnicodeTables::LineBreak_ZWJ ? ncls : QUnicodeTables::LineBreak_AL]) { |
1057 | case LB::DirectBreak: |
1058 | attributes[pos].lineBreak = true; |
1059 | break; |
1060 | case LB::IndirectBreak: |
1061 | if (lcls == QUnicodeTables::LineBreak_SP) |
1062 | attributes[pos].lineBreak = true; |
1063 | break; |
1064 | case LB::CombiningIndirectBreak: |
1065 | if (lcls != QUnicodeTables::LineBreak_SP) |
1066 | goto next_no_cls_update; |
1067 | attributes[pos].lineBreak = true; |
1068 | break; |
1069 | case LB::CombiningProhibitedBreak: |
1070 | if (lcls != QUnicodeTables::LineBreak_SP) |
1071 | goto next_no_cls_update; |
1072 | break; |
1073 | case LB::ProhibitedBreakAfterHebrewPlusHyphen: |
1074 | if (lcls != QUnicodeTables::LineBreak_HL) |
1075 | attributes[pos].lineBreak = true; |
1076 | break; |
1077 | case LB::IndirectBreakIfNarrow: |
1078 | using EAW = QUnicodeTables::EastAsianWidth; |
1079 | switch (EAW(prop->eastAsianWidth)) { |
1080 | default: |
1081 | if (lcls != QUnicodeTables::LineBreak_SP) |
1082 | break; |
1083 | Q_FALLTHROUGH(); |
1084 | case QUnicodeTables::EastAsianWidth::F: |
1085 | case QUnicodeTables::EastAsianWidth::W: |
1086 | case QUnicodeTables::EastAsianWidth::H: |
1087 | attributes[pos].lineBreak = true; |
1088 | break; |
1089 | } |
1090 | break; |
1091 | case LB::DirectBreakOutsideNumericSequence: |
1092 | if (neactlast == LB::NS::None || neactlast > LB::NS::Break) |
1093 | attributes[pos].lineBreak = true; |
1094 | break; |
1095 | case LB::ProhibitedBreak: |
1096 | // nothing to do |
1097 | default: |
1098 | break; |
1099 | } |
1100 | |
1101 | next: |
1102 | if (ncls != QUnicodeTables::LineBreak_CM && ncls != QUnicodeTables::LineBreak_ZWJ) { |
1103 | cls = ncls; |
1104 | lastProp = prop; |
1105 | } |
1106 | next_no_cls_update: |
1107 | lcls = ncls; |
1108 | } |
1109 | |
1110 | if (Q_UNLIKELY(LB::NS::actionTable[nelast][LB::NS::XX] == LB::NS::Break)) { |
1111 | // LB25: do not break lines inside numbers |
1112 | for (qsizetype j = nestart + 1; j < len; ++j) |
1113 | attributes[j].lineBreak = false; |
1114 | } |
1115 | |
1116 | attributes[0].lineBreak = attributes[0].mandatoryBreak = false; // LB2 |
1117 | attributes[len].lineBreak = attributes[len].mandatoryBreak = true; // LB3 |
1118 | } |
1119 | |
1120 | |
1121 | static void getWhiteSpaces(const char16_t *string, qsizetype len, QCharAttributes *attributes) |
1122 | { |
1123 | for (qsizetype i = 0; i != len; ++i) { |
1124 | uint ucs4 = string[i]; |
1125 | if (QChar::isHighSurrogate(ucs4) && i + 1 != len) { |
1126 | ushort low = string[i + 1]; |
1127 | if (QChar::isLowSurrogate(ucs4: low)) { |
1128 | ucs4 = QChar::surrogateToUcs4(high: ucs4, low); |
1129 | ++i; |
1130 | } |
1131 | } |
1132 | |
1133 | if (Q_UNLIKELY(QChar::isSpace(ucs4))) |
1134 | attributes[i].whiteSpace = true; |
1135 | } |
1136 | } |
1137 | |
1138 | namespace Tailored { |
1139 | |
1140 | using CharAttributeFunction = void (*)(QChar::Script script, const char16_t *text, qsizetype from, qsizetype len, QCharAttributes *attributes); |
1141 | |
1142 | |
1143 | enum Form { |
1144 | Invalid = 0x0, |
1145 | UnknownForm = Invalid, |
1146 | Consonant, |
1147 | Nukta, |
1148 | Halant, |
1149 | Matra, |
1150 | VowelMark, |
1151 | StressMark, |
1152 | IndependentVowel, |
1153 | LengthMark, |
1154 | Control, |
1155 | Other |
1156 | }; |
1157 | |
1158 | static const unsigned char indicForms[0xe00-0x900] = { |
1159 | // Devangari |
1160 | Invalid, VowelMark, VowelMark, VowelMark, |
1161 | IndependentVowel, IndependentVowel, IndependentVowel, IndependentVowel, |
1162 | IndependentVowel, IndependentVowel, IndependentVowel, IndependentVowel, |
1163 | IndependentVowel, IndependentVowel, IndependentVowel, IndependentVowel, |
1164 | |
1165 | IndependentVowel, IndependentVowel, IndependentVowel, IndependentVowel, |
1166 | IndependentVowel, Consonant, Consonant, Consonant, |
1167 | Consonant, Consonant, Consonant, Consonant, |
1168 | Consonant, Consonant, Consonant, Consonant, |
1169 | |
1170 | Consonant, Consonant, Consonant, Consonant, |
1171 | Consonant, Consonant, Consonant, Consonant, |
1172 | Consonant, Consonant, Consonant, Consonant, |
1173 | Consonant, Consonant, Consonant, Consonant, |
1174 | |
1175 | Consonant, Consonant, Consonant, Consonant, |
1176 | Consonant, Consonant, Consonant, Consonant, |
1177 | Consonant, Consonant, UnknownForm, UnknownForm, |
1178 | Nukta, Other, Matra, Matra, |
1179 | |
1180 | Matra, Matra, Matra, Matra, |
1181 | Matra, Matra, Matra, Matra, |
1182 | Matra, Matra, Matra, Matra, |
1183 | Matra, Halant, UnknownForm, UnknownForm, |
1184 | |
1185 | Other, StressMark, StressMark, StressMark, |
1186 | StressMark, UnknownForm, UnknownForm, UnknownForm, |
1187 | Consonant, Consonant, Consonant, Consonant, |
1188 | Consonant, Consonant, Consonant, Consonant, |
1189 | |
1190 | IndependentVowel, IndependentVowel, VowelMark, VowelMark, |
1191 | Other, Other, Other, Other, |
1192 | Other, Other, Other, Other, |
1193 | Other, Other, Other, Other, |
1194 | |
1195 | Other, Other, Other, Other, |
1196 | Other, Other, Other, Other, |
1197 | Other, Other, Other, Consonant, |
1198 | Consonant, Consonant /* ??? */, Consonant, Consonant, |
1199 | |
1200 | // Bengali |
1201 | Invalid, VowelMark, VowelMark, VowelMark, |
1202 | Invalid, IndependentVowel, IndependentVowel, IndependentVowel, |
1203 | IndependentVowel, IndependentVowel, IndependentVowel, IndependentVowel, |
1204 | IndependentVowel, Invalid, Invalid, IndependentVowel, |
1205 | |
1206 | IndependentVowel, Invalid, Invalid, IndependentVowel, |
1207 | IndependentVowel, Consonant, Consonant, Consonant, |
1208 | Consonant, Consonant, Consonant, Consonant, |
1209 | Consonant, Consonant, Consonant, Consonant, |
1210 | |
1211 | Consonant, Consonant, Consonant, Consonant, |
1212 | Consonant, Consonant, Consonant, Consonant, |
1213 | Consonant, Invalid, Consonant, Consonant, |
1214 | Consonant, Consonant, Consonant, Consonant, |
1215 | |
1216 | Consonant, Invalid, Consonant, Invalid, |
1217 | Invalid, Invalid, Consonant, Consonant, |
1218 | Consonant, Consonant, UnknownForm, UnknownForm, |
1219 | Nukta, Other, Matra, Matra, |
1220 | |
1221 | Matra, Matra, Matra, Matra, |
1222 | Matra, Invalid, Invalid, Matra, |
1223 | Matra, Invalid, Invalid, Matra, |
1224 | Matra, Halant, Consonant, UnknownForm, |
1225 | |
1226 | Invalid, Invalid, Invalid, Invalid, |
1227 | Invalid, Invalid, Invalid, VowelMark, |
1228 | Invalid, Invalid, Invalid, Invalid, |
1229 | Consonant, Consonant, Invalid, Consonant, |
1230 | |
1231 | IndependentVowel, IndependentVowel, VowelMark, VowelMark, |
1232 | Other, Other, Other, Other, |
1233 | Other, Other, Other, Other, |
1234 | Other, Other, Other, Other, |
1235 | |
1236 | Consonant, Consonant, Other, Other, |
1237 | Other, Other, Other, Other, |
1238 | Other, Other, Other, Other, |
1239 | Other, Other, Other, Other, |
1240 | |
1241 | // Gurmukhi |
1242 | Invalid, VowelMark, VowelMark, VowelMark, |
1243 | Invalid, IndependentVowel, IndependentVowel, IndependentVowel, |
1244 | IndependentVowel, IndependentVowel, IndependentVowel, Invalid, |
1245 | Invalid, Invalid, Invalid, IndependentVowel, |
1246 | |
1247 | IndependentVowel, Invalid, Invalid, IndependentVowel, |
1248 | IndependentVowel, Consonant, Consonant, Consonant, |
1249 | Consonant, Consonant, Consonant, Consonant, |
1250 | Consonant, Consonant, Consonant, Consonant, |
1251 | |
1252 | Consonant, Consonant, Consonant, Consonant, |
1253 | Consonant, Consonant, Consonant, Consonant, |
1254 | Consonant, Invalid, Consonant, Consonant, |
1255 | Consonant, Consonant, Consonant, Consonant, |
1256 | |
1257 | Consonant, Invalid, Consonant, Consonant, |
1258 | Invalid, Consonant, Consonant, Invalid, |
1259 | Consonant, Consonant, UnknownForm, UnknownForm, |
1260 | Nukta, Other, Matra, Matra, |
1261 | |
1262 | Matra, Matra, Matra, Invalid, |
1263 | Invalid, Invalid, Invalid, Matra, |
1264 | Matra, Invalid, Invalid, Matra, |
1265 | Matra, Halant, UnknownForm, UnknownForm, |
1266 | |
1267 | Invalid, Invalid, Invalid, Invalid, |
1268 | Invalid, UnknownForm, UnknownForm, UnknownForm, |
1269 | Invalid, Consonant, Consonant, Consonant, |
1270 | Consonant, Invalid, Consonant, Invalid, |
1271 | |
1272 | Other, Other, Invalid, Invalid, |
1273 | Other, Other, Other, Other, |
1274 | Other, Other, Other, Other, |
1275 | Other, Other, Other, Other, |
1276 | |
1277 | StressMark, StressMark, Consonant, Consonant, |
1278 | Other, Other, Other, Other, |
1279 | Other, Other, Other, Other, |
1280 | Other, Other, Other, Other, |
1281 | |
1282 | // Gujarati |
1283 | Invalid, VowelMark, VowelMark, VowelMark, |
1284 | Invalid, IndependentVowel, IndependentVowel, IndependentVowel, |
1285 | IndependentVowel, IndependentVowel, IndependentVowel, IndependentVowel, |
1286 | IndependentVowel, IndependentVowel, Invalid, IndependentVowel, |
1287 | |
1288 | IndependentVowel, IndependentVowel, Invalid, IndependentVowel, |
1289 | IndependentVowel, Consonant, Consonant, Consonant, |
1290 | Consonant, Consonant, Consonant, Consonant, |
1291 | Consonant, Consonant, Consonant, Consonant, |
1292 | |
1293 | Consonant, Consonant, Consonant, Consonant, |
1294 | Consonant, Consonant, Consonant, Consonant, |
1295 | Consonant, Invalid, Consonant, Consonant, |
1296 | Consonant, Consonant, Consonant, Consonant, |
1297 | |
1298 | Consonant, Invalid, Consonant, Consonant, |
1299 | Invalid, Consonant, Consonant, Consonant, |
1300 | Consonant, Consonant, UnknownForm, UnknownForm, |
1301 | Nukta, Other, Matra, Matra, |
1302 | |
1303 | Matra, Matra, Matra, Matra, |
1304 | Matra, Matra, Invalid, Matra, |
1305 | Matra, Matra, Invalid, Matra, |
1306 | Matra, Halant, UnknownForm, UnknownForm, |
1307 | |
1308 | Other, UnknownForm, UnknownForm, UnknownForm, |
1309 | UnknownForm, UnknownForm, UnknownForm, UnknownForm, |
1310 | UnknownForm, UnknownForm, UnknownForm, UnknownForm, |
1311 | UnknownForm, UnknownForm, UnknownForm, UnknownForm, |
1312 | |
1313 | IndependentVowel, IndependentVowel, VowelMark, VowelMark, |
1314 | Other, Other, Other, Other, |
1315 | Other, Other, Other, Other, |
1316 | Other, Other, Other, Other, |
1317 | |
1318 | Other, Other, Other, Other, |
1319 | Other, Other, Other, Other, |
1320 | Other, Other, Other, Other, |
1321 | Other, Other, Other, Other, |
1322 | |
1323 | // Oriya |
1324 | Invalid, VowelMark, VowelMark, VowelMark, |
1325 | Invalid, IndependentVowel, IndependentVowel, IndependentVowel, |
1326 | IndependentVowel, IndependentVowel, IndependentVowel, IndependentVowel, |
1327 | IndependentVowel, Invalid, Invalid, IndependentVowel, |
1328 | |
1329 | IndependentVowel, Invalid, Invalid, IndependentVowel, |
1330 | IndependentVowel, Consonant, Consonant, Consonant, |
1331 | Consonant, Consonant, Consonant, Consonant, |
1332 | Consonant, Consonant, Consonant, Consonant, |
1333 | |
1334 | Consonant, Consonant, Consonant, Consonant, |
1335 | Consonant, Consonant, Consonant, Consonant, |
1336 | Consonant, Invalid, Consonant, Consonant, |
1337 | Consonant, Consonant, Consonant, Consonant, |
1338 | |
1339 | Consonant, Invalid, Consonant, Consonant, |
1340 | Invalid, Consonant, Consonant, Consonant, |
1341 | Consonant, Consonant, UnknownForm, UnknownForm, |
1342 | Nukta, Other, Matra, Matra, |
1343 | |
1344 | Matra, Matra, Matra, Matra, |
1345 | Invalid, Invalid, Invalid, Matra, |
1346 | Matra, Invalid, Invalid, Matra, |
1347 | Matra, Halant, UnknownForm, UnknownForm, |
1348 | |
1349 | Other, Invalid, Invalid, Invalid, |
1350 | Invalid, UnknownForm, LengthMark, LengthMark, |
1351 | Invalid, Invalid, Invalid, Invalid, |
1352 | Consonant, Consonant, Invalid, Consonant, |
1353 | |
1354 | IndependentVowel, IndependentVowel, Invalid, Invalid, |
1355 | Invalid, Invalid, Other, Other, |
1356 | Other, Other, Other, Other, |
1357 | Other, Other, Other, Other, |
1358 | |
1359 | Other, Consonant, Other, Other, |
1360 | Other, Other, Other, Other, |
1361 | Other, Other, Other, Other, |
1362 | Other, Other, Other, Other, |
1363 | |
1364 | //Tamil |
1365 | Invalid, Invalid, VowelMark, Other, |
1366 | Invalid, IndependentVowel, IndependentVowel, IndependentVowel, |
1367 | IndependentVowel, IndependentVowel, IndependentVowel, Invalid, |
1368 | Invalid, Invalid, IndependentVowel, IndependentVowel, |
1369 | |
1370 | IndependentVowel, Invalid, IndependentVowel, IndependentVowel, |
1371 | IndependentVowel, Consonant, Invalid, Invalid, |
1372 | Invalid, Consonant, Consonant, Invalid, |
1373 | Consonant, Invalid, Consonant, Consonant, |
1374 | |
1375 | Invalid, Invalid, Invalid, Consonant, |
1376 | Consonant, Invalid, Invalid, Invalid, |
1377 | Consonant, Consonant, Consonant, Invalid, |
1378 | Invalid, Invalid, Consonant, Consonant, |
1379 | |
1380 | Consonant, Consonant, Consonant, Consonant, |
1381 | Consonant, Consonant, Consonant, Consonant, |
1382 | Consonant, Consonant, UnknownForm, UnknownForm, |
1383 | Invalid, Invalid, Matra, Matra, |
1384 | |
1385 | Matra, Matra, Matra, Invalid, |
1386 | Invalid, Invalid, Matra, Matra, |
1387 | Matra, Invalid, Matra, Matra, |
1388 | Matra, Halant, Invalid, Invalid, |
1389 | |
1390 | Invalid, Invalid, Invalid, Invalid, |
1391 | Invalid, Invalid, Invalid, LengthMark, |
1392 | Invalid, Invalid, Invalid, Invalid, |
1393 | Invalid, Invalid, Invalid, Invalid, |
1394 | |
1395 | Invalid, Invalid, Invalid, Invalid, |
1396 | Invalid, Invalid, Other, Other, |
1397 | Other, Other, Other, Other, |
1398 | Other, Other, Other, Other, |
1399 | |
1400 | Other, Other, Other, Other, |
1401 | Other, Other, Other, Other, |
1402 | Other, Other, Other, Other, |
1403 | Other, Other, Other, Other, |
1404 | |
1405 | // Telugu |
1406 | Invalid, VowelMark, VowelMark, VowelMark, |
1407 | Invalid, IndependentVowel, IndependentVowel, IndependentVowel, |
1408 | IndependentVowel, IndependentVowel, IndependentVowel, IndependentVowel, |
1409 | IndependentVowel, Invalid, IndependentVowel, IndependentVowel, |
1410 | |
1411 | IndependentVowel, Invalid, IndependentVowel, IndependentVowel, |
1412 | IndependentVowel, Consonant, Consonant, Consonant, |
1413 | Consonant, Consonant, Consonant, Consonant, |
1414 | Consonant, Consonant, Consonant, Consonant, |
1415 | |
1416 | Consonant, Consonant, Consonant, Consonant, |
1417 | Consonant, Consonant, Consonant, Consonant, |
1418 | Consonant, Invalid, Consonant, Consonant, |
1419 | Consonant, Consonant, Consonant, Consonant, |
1420 | |
1421 | Consonant, Consonant, Consonant, Consonant, |
1422 | Invalid, Consonant, Consonant, Consonant, |
1423 | Consonant, Consonant, UnknownForm, UnknownForm, |
1424 | Invalid, Invalid, Matra, Matra, |
1425 | |
1426 | Matra, Matra, Matra, Matra, |
1427 | Matra, Invalid, Matra, Matra, |
1428 | Matra, Invalid, Matra, Matra, |
1429 | Matra, Halant, Invalid, Invalid, |
1430 | |
1431 | Invalid, Invalid, Invalid, Invalid, |
1432 | Invalid, LengthMark, Matra, Invalid, |
1433 | Invalid, Invalid, Invalid, Invalid, |
1434 | Invalid, Invalid, Invalid, Invalid, |
1435 | |
1436 | IndependentVowel, IndependentVowel, Invalid, Invalid, |
1437 | Invalid, Invalid, Other, Other, |
1438 | Other, Other, Other, Other, |
1439 | Other, Other, Other, Other, |
1440 | |
1441 | Other, Other, Other, Other, |
1442 | Other, Other, Other, Other, |
1443 | Other, Other, Other, Other, |
1444 | Other, Other, Other, Other, |
1445 | |
1446 | // Kannada |
1447 | Invalid, Invalid, VowelMark, VowelMark, |
1448 | Invalid, IndependentVowel, IndependentVowel, IndependentVowel, |
1449 | IndependentVowel, IndependentVowel, IndependentVowel, IndependentVowel, |
1450 | IndependentVowel, Invalid, IndependentVowel, IndependentVowel, |
1451 | |
1452 | IndependentVowel, Invalid, IndependentVowel, IndependentVowel, |
1453 | IndependentVowel, Consonant, Consonant, Consonant, |
1454 | Consonant, Consonant, Consonant, Consonant, |
1455 | Consonant, Consonant, Consonant, Consonant, |
1456 | |
1457 | Consonant, Consonant, Consonant, Consonant, |
1458 | Consonant, Consonant, Consonant, Consonant, |
1459 | Consonant, Invalid, Consonant, Consonant, |
1460 | Consonant, Consonant, Consonant, Consonant, |
1461 | |
1462 | Consonant, Consonant, Consonant, Consonant, |
1463 | Invalid, Consonant, Consonant, Consonant, |
1464 | Consonant, Consonant, UnknownForm, UnknownForm, |
1465 | Nukta, Other, Matra, Matra, |
1466 | |
1467 | Matra, Matra, Matra, Matra, |
1468 | Matra, Invalid, Matra, Matra, |
1469 | Matra, Invalid, Matra, Matra, |
1470 | Matra, Halant, Invalid, Invalid, |
1471 | |
1472 | Invalid, Invalid, Invalid, Invalid, |
1473 | Invalid, LengthMark, LengthMark, Invalid, |
1474 | Invalid, Invalid, Invalid, Invalid, |
1475 | Invalid, Invalid, Consonant, Invalid, |
1476 | |
1477 | IndependentVowel, IndependentVowel, VowelMark, VowelMark, |
1478 | Invalid, Invalid, Other, Other, |
1479 | Other, Other, Other, Other, |
1480 | Other, Other, Other, Other, |
1481 | |
1482 | Other, Other, Other, Other, |
1483 | Other, Other, Other, Other, |
1484 | Other, Other, Other, Other, |
1485 | Other, Other, Other, Other, |
1486 | |
1487 | // Malayalam |
1488 | Invalid, Invalid, VowelMark, VowelMark, |
1489 | Invalid, IndependentVowel, IndependentVowel, IndependentVowel, |
1490 | IndependentVowel, IndependentVowel, IndependentVowel, IndependentVowel, |
1491 | IndependentVowel, Invalid, IndependentVowel, IndependentVowel, |
1492 | |
1493 | IndependentVowel, Invalid, IndependentVowel, IndependentVowel, |
1494 | IndependentVowel, Consonant, Consonant, Consonant, |
1495 | Consonant, Consonant, Consonant, Consonant, |
1496 | Consonant, Consonant, Consonant, Consonant, |
1497 | |
1498 | Consonant, Consonant, Consonant, Consonant, |
1499 | Consonant, Consonant, Consonant, Consonant, |
1500 | Consonant, Invalid, Consonant, Consonant, |
1501 | Consonant, Consonant, Consonant, Consonant, |
1502 | |
1503 | Consonant, Consonant, Consonant, Consonant, |
1504 | Consonant, Consonant, Consonant, Consonant, |
1505 | Consonant, Consonant, UnknownForm, UnknownForm, |
1506 | Invalid, Invalid, Matra, Matra, |
1507 | |
1508 | Matra, Matra, Matra, Matra, |
1509 | Invalid, Invalid, Matra, Matra, |
1510 | Matra, Invalid, Matra, Matra, |
1511 | Matra, Halant, Invalid, Invalid, |
1512 | |
1513 | Invalid, Invalid, Invalid, Invalid, |
1514 | Invalid, Invalid, Invalid, Matra, |
1515 | Invalid, Invalid, Invalid, Invalid, |
1516 | Invalid, Invalid, Invalid, Invalid, |
1517 | |
1518 | IndependentVowel, IndependentVowel, Invalid, Invalid, |
1519 | Invalid, Invalid, Other, Other, |
1520 | Other, Other, Other, Other, |
1521 | Other, Other, Other, Other, |
1522 | |
1523 | Other, Other, Other, Other, |
1524 | Other, Other, Other, Other, |
1525 | Other, Other, Other, Other, |
1526 | Other, Other, Other, Other, |
1527 | |
1528 | // Sinhala |
1529 | Invalid, Invalid, VowelMark, VowelMark, |
1530 | Invalid, IndependentVowel, IndependentVowel, IndependentVowel, |
1531 | IndependentVowel, IndependentVowel, IndependentVowel, IndependentVowel, |
1532 | IndependentVowel, IndependentVowel, IndependentVowel, IndependentVowel, |
1533 | |
1534 | IndependentVowel, IndependentVowel, IndependentVowel, IndependentVowel, |
1535 | IndependentVowel, IndependentVowel, IndependentVowel, Invalid, |
1536 | Invalid, Invalid, Consonant, Consonant, |
1537 | Consonant, Consonant, Consonant, Consonant, |
1538 | |
1539 | Consonant, Consonant, Consonant, Consonant, |
1540 | Consonant, Consonant, Consonant, Consonant, |
1541 | Consonant, Consonant, Consonant, Consonant, |
1542 | Consonant, Consonant, Consonant, Consonant, |
1543 | |
1544 | Consonant, Consonant, Invalid, Consonant, |
1545 | Consonant, Consonant, Consonant, Consonant, |
1546 | Consonant, Consonant, Consonant, Consonant, |
1547 | Invalid, Consonant, Invalid, Invalid, |
1548 | |
1549 | Consonant, Consonant, Consonant, Consonant, |
1550 | Consonant, Consonant, Consonant, Invalid, |
1551 | Invalid, Invalid, Halant, Invalid, |
1552 | Invalid, Invalid, Invalid, Matra, |
1553 | |
1554 | Matra, Matra, Matra, Matra, |
1555 | Matra, Invalid, Matra, Invalid, |
1556 | Matra, Matra, Matra, Matra, |
1557 | Matra, Matra, Matra, Matra, |
1558 | |
1559 | Invalid, Invalid, Invalid, Invalid, |
1560 | Invalid, Invalid, Invalid, Invalid, |
1561 | Invalid, Invalid, Invalid, Invalid, |
1562 | Invalid, Invalid, Invalid, Invalid, |
1563 | |
1564 | Invalid, Invalid, Matra, Matra, |
1565 | Other, Other, Other, Other, |
1566 | Other, Other, Other, Other, |
1567 | Other, Other, Other, Other, |
1568 | }; |
1569 | |
1570 | static inline Form form(unsigned short uc) { |
1571 | if (uc < 0x900 || uc > 0xdff) { |
1572 | if (uc == 0x25cc) |
1573 | return Consonant; |
1574 | if (uc == 0x200c || uc == 0x200d) |
1575 | return Control; |
1576 | return Other; |
1577 | } |
1578 | return (Form)indicForms[uc-0x900]; |
1579 | } |
1580 | |
1581 | // #define INDIC_DEBUG |
1582 | #ifdef INDIC_DEBUG |
1583 | #define IDEBUG qDebug |
1584 | #else |
1585 | #define IDEBUG if constexpr (1) ; else qDebug |
1586 | #endif |
1587 | |
1588 | /* syllables are of the form: |
1589 | |
1590 | (Consonant Nukta? Halant)* Consonant Matra? VowelMark? StressMark? |
1591 | (Consonant Nukta? Halant)* Consonant Halant |
1592 | IndependentVowel VowelMark? StressMark? |
1593 | |
1594 | We return syllable boundaries on invalid combinations as well |
1595 | */ |
1596 | static qsizetype indic_nextSyllableBoundary(QChar::Script script, const char16_t *s, qsizetype start, qsizetype end, bool *invalid) |
1597 | { |
1598 | *invalid = false; |
1599 | IDEBUG(msg: "indic_nextSyllableBoundary: start=%lld, end=%lld", qlonglong(start), qlonglong(end)); |
1600 | const char16_t *uc = s+start; |
1601 | |
1602 | qsizetype pos = 0; |
1603 | Form state = form(uc: uc[pos]); |
1604 | IDEBUG(msg: "state[%lld]=%d (uc=%4x)", qlonglong(pos), state, uc[pos]); |
1605 | pos++; |
1606 | |
1607 | if (state != Consonant && state != IndependentVowel) { |
1608 | if (state != Other) |
1609 | *invalid = true; |
1610 | goto finish; |
1611 | } |
1612 | |
1613 | while (pos < end - start) { |
1614 | Form newState = form(uc: uc[pos]); |
1615 | IDEBUG(msg: "state[%lld]=%d (uc=%4x)", qlonglong(pos), newState, uc[pos]); |
1616 | switch (newState) { |
1617 | case Control: |
1618 | newState = state; |
1619 | if (state == Halant && uc[pos] == 0x200d /* ZWJ */) |
1620 | break; |
1621 | // the control character should be the last char in the item |
1622 | if (state == Consonant && script == QChar::Script_Bengali && uc[pos-1] == 0x09B0 && uc[pos] == 0x200d /* ZWJ */) |
1623 | break; |
1624 | if (state == Consonant && script == QChar::Script_Kannada && uc[pos-1] == 0x0CB0 && uc[pos] == 0x200d /* ZWJ */) |
1625 | break; |
1626 | // Bengali and Kannada has a special exception for rendering yaphala with ra (to avoid reph) see http://www.unicode.org/faq/indic.html#15 |
1627 | ++pos; |
1628 | goto finish; |
1629 | case Consonant: |
1630 | if (state == Halant && (script != QChar::Script_Sinhala || uc[pos-1] == 0x200d /* ZWJ */)) |
1631 | break; |
1632 | goto finish; |
1633 | case Halant: |
1634 | if (state == Nukta || state == Consonant) |
1635 | break; |
1636 | // Bengali has a special exception allowing the combination Vowel_A/E + Halant + Ya |
1637 | if (script == QChar::Script_Bengali && pos == 1 && |
1638 | (uc[0] == 0x0985 || uc[0] == 0x098f)) |
1639 | break; |
1640 | // Sinhala uses the Halant as a component of certain matras. Allow these, but keep the state on Matra. |
1641 | if (script == QChar::Script_Sinhala && state == Matra) { |
1642 | ++pos; |
1643 | continue; |
1644 | } |
1645 | if (script == QChar::Script_Malayalam && state == Matra && uc[pos-1] == 0x0d41) { |
1646 | ++pos; |
1647 | continue; |
1648 | } |
1649 | goto finish; |
1650 | case Nukta: |
1651 | if (state == Consonant) |
1652 | break; |
1653 | goto finish; |
1654 | case StressMark: |
1655 | if (state == VowelMark) |
1656 | break; |
1657 | Q_FALLTHROUGH(); |
1658 | case VowelMark: |
1659 | if (state == Matra || state == LengthMark || state == IndependentVowel) |
1660 | break; |
1661 | Q_FALLTHROUGH(); |
1662 | case Matra: |
1663 | if (state == Consonant || state == Nukta) |
1664 | break; |
1665 | if (state == Matra) { |
1666 | // ### needs proper testing for correct two/three part matras |
1667 | break; |
1668 | } |
1669 | // ### not sure if this is correct. If it is, does it apply only to Bengali or should |
1670 | // it work for all Indic languages? |
1671 | // the combination Independent_A + Vowel Sign AA is allowed. |
1672 | if (script == QChar::Script_Bengali && uc[pos] == 0x9be && uc[pos-1] == 0x985) |
1673 | break; |
1674 | if (script == QChar::Script_Tamil && state == Matra) { |
1675 | if (uc[pos-1] == 0x0bc6 && |
1676 | (uc[pos] == 0xbbe || uc[pos] == 0xbd7)) |
1677 | break; |
1678 | if (uc[pos-1] == 0x0bc7 && uc[pos] == 0xbbe) |
1679 | break; |
1680 | } |
1681 | goto finish; |
1682 | |
1683 | case LengthMark: |
1684 | if (state == Matra) { |
1685 | // ### needs proper testing for correct two/three part matras |
1686 | break; |
1687 | } |
1688 | Q_FALLTHROUGH(); |
1689 | case IndependentVowel: |
1690 | case Invalid: |
1691 | case Other: |
1692 | goto finish; |
1693 | } |
1694 | state = newState; |
1695 | pos++; |
1696 | } |
1697 | finish: |
1698 | return pos+start; |
1699 | } |
1700 | |
1701 | static void indicAttributes(QChar::Script script, const char16_t *text, qsizetype from, qsizetype len, QCharAttributes *attributes) |
1702 | { |
1703 | qsizetype end = from + len; |
1704 | attributes += from; |
1705 | qsizetype i = 0; |
1706 | while (i < len) { |
1707 | bool invalid; |
1708 | qsizetype boundary = indic_nextSyllableBoundary(script, s: text, start: from+i, end, invalid: &invalid) - from; |
1709 | attributes[i].graphemeBoundary = true; |
1710 | |
1711 | if (boundary > len-1) boundary = len; |
1712 | i++; |
1713 | while (i < boundary) { |
1714 | attributes[i].graphemeBoundary = false; |
1715 | ++i; |
1716 | } |
1717 | assert(i == boundary); |
1718 | } |
1719 | |
1720 | |
1721 | } |
1722 | |
1723 | #if QT_CONFIG(library) |
1724 | |
1725 | #define LIBTHAI_MAJOR 0 |
1726 | |
1727 | /* |
1728 | * if libthai changed please update these codes too. |
1729 | */ |
1730 | struct thcell_t { |
1731 | unsigned char base; /**< base character */ |
1732 | unsigned char hilo; /**< upper/lower vowel/diacritic */ |
1733 | unsigned char top; /**< top-level mark */ |
1734 | }; |
1735 | |
1736 | using ThBrk = struct _ThBrk; |
1737 | |
1738 | namespace { |
1739 | |
1740 | class LibThai final |
1741 | { |
1742 | Q_DISABLE_COPY_MOVE(LibThai) |
1743 | |
1744 | using th_brk_new_def = ThBrk *(*)(const char *); |
1745 | using th_brk_delete_def = void (*)(ThBrk *); |
1746 | using th_brk_find_breaks_def = int (*)(ThBrk *, const unsigned char *, int *, size_t); |
1747 | using th_next_cell_def = size_t (*)(const unsigned char *, size_t, struct thcell_t *, int); |
1748 | |
1749 | public: |
1750 | LibThai() : m_library("thai"_L1, LIBTHAI_MAJOR) |
1751 | { |
1752 | m_th_brk_find_breaks = |
1753 | reinterpret_cast<th_brk_find_breaks_def>(m_library.resolve(symbol: "th_brk_find_breaks")); |
1754 | m_th_next_cell = reinterpret_cast<th_next_cell_def>(m_library.resolve(symbol: "th_next_cell")); |
1755 | |
1756 | auto th_brk_new = reinterpret_cast<th_brk_new_def>(m_library.resolve(symbol: "th_brk_new")); |
1757 | if (th_brk_new) { |
1758 | m_state = th_brk_new(nullptr); |
1759 | m_th_brk_delete = |
1760 | reinterpret_cast<th_brk_delete_def>(m_library.resolve(symbol: "th_brk_delete")); |
1761 | } |
1762 | } |
1763 | |
1764 | ~LibThai() |
1765 | { |
1766 | if (m_state && m_th_brk_delete) |
1767 | m_th_brk_delete(m_state); |
1768 | m_library.unload(); |
1769 | } |
1770 | |
1771 | bool isInitialized() const { return m_th_brk_find_breaks && m_th_next_cell && m_state; } |
1772 | |
1773 | int brk_find_breaks(const unsigned char *s, int *pos, size_t pos_sz) const |
1774 | { |
1775 | Q_ASSERT(m_state); |
1776 | Q_ASSERT(m_th_brk_find_breaks); |
1777 | return m_th_brk_find_breaks(m_state, s, pos, pos_sz); |
1778 | } |
1779 | |
1780 | size_t next_cell(const unsigned char *s, size_t len, struct thcell_t *cell, int is_decomp_am) |
1781 | { |
1782 | Q_ASSERT(m_th_next_cell); |
1783 | return m_th_next_cell(s, len, cell, is_decomp_am); |
1784 | } |
1785 | |
1786 | private: |
1787 | QLibrary m_library; |
1788 | |
1789 | // Global state for th_brk_find_breaks(). |
1790 | // Note: even if signature for th_brk_find_breaks() suggests otherwise, the |
1791 | // state is read-only, and so it is safe to use it from multiple threads after |
1792 | // initialization. This is also stated in the libthai documentation. |
1793 | ThBrk *m_state = nullptr; |
1794 | |
1795 | th_brk_find_breaks_def m_th_brk_find_breaks = nullptr; |
1796 | th_next_cell_def m_th_next_cell = nullptr; |
1797 | th_brk_delete_def m_th_brk_delete = nullptr; |
1798 | }; |
1799 | |
1800 | } // unnamed namespace |
1801 | |
1802 | Q_GLOBAL_STATIC(LibThai, g_libThai) |
1803 | |
1804 | static void to_tis620(const char16_t *string, qsizetype len, char *cstr) |
1805 | { |
1806 | qsizetype i; |
1807 | unsigned char *result = reinterpret_cast<unsigned char *>(cstr); |
1808 | |
1809 | for (i = 0; i < len; ++i) { |
1810 | if (string[i] <= 0xa0) |
1811 | result[i] = static_cast<unsigned char>(string[i]); |
1812 | else if (string[i] >= 0xe01 && string[i] <= 0xe5b) |
1813 | result[i] = static_cast<unsigned char>(string[i] - 0xe00 + 0xa0); |
1814 | else |
1815 | result[i] = static_cast<unsigned char>(~0); // Same encoding as libthai uses for invalid chars |
1816 | } |
1817 | |
1818 | result[len] = 0; |
1819 | } |
1820 | |
1821 | /* |
1822 | * Thai Attributes: computes Word Break, Word Boundary and Char stop for THAI. |
1823 | */ |
1824 | static void thaiAssignAttributes(const char16_t *string, qsizetype len, QCharAttributes *attributes) |
1825 | { |
1826 | constexpr qsizetype Prealloc = 128; |
1827 | QVarLengthArray<char, Prealloc + 1> s(len + 1); |
1828 | QVarLengthArray<int, Prealloc> break_positions(len); |
1829 | qsizetype numbreaks, i; |
1830 | struct thcell_t tis_cell; |
1831 | |
1832 | LibThai *libThai = g_libThai; |
1833 | if (!libThai || !libThai->isInitialized()) |
1834 | return; |
1835 | |
1836 | to_tis620(string, len, cstr: s.data()); |
1837 | |
1838 | for (i = 0; i < len; ++i) { |
1839 | attributes[i].wordBreak = false; |
1840 | attributes[i].wordStart = false; |
1841 | attributes[i].wordEnd = false; |
1842 | attributes[i].lineBreak = false; |
1843 | } |
1844 | |
1845 | attributes[0].wordBreak = true; |
1846 | attributes[0].wordStart = true; |
1847 | attributes[0].wordEnd = false; |
1848 | numbreaks = libThai->brk_find_breaks(s: reinterpret_cast<const unsigned char *>(s.data()), |
1849 | pos: break_positions.data(), |
1850 | pos_sz: static_cast<size_t>(break_positions.size())); |
1851 | for (i = 0; i < numbreaks; ++i) { |
1852 | attributes[break_positions[i]].wordBreak = true; |
1853 | attributes[break_positions[i]].wordStart = true; |
1854 | attributes[break_positions[i]].wordEnd = true; |
1855 | attributes[break_positions[i]].lineBreak = true; |
1856 | } |
1857 | if (numbreaks > 0) |
1858 | attributes[break_positions[numbreaks - 1]].wordStart = false; |
1859 | |
1860 | /* manage grapheme boundaries */ |
1861 | i = 0; |
1862 | while (i < len) { |
1863 | size_t cell_length = |
1864 | libThai->next_cell(s: reinterpret_cast<const unsigned char *>(s.data()) + i, |
1865 | len: size_t(len - i), cell: &tis_cell, is_decomp_am: true); |
1866 | |
1867 | attributes[i].graphemeBoundary = true; |
1868 | for (size_t j = 1; j < cell_length; ++j) |
1869 | attributes[i + j].graphemeBoundary = false; |
1870 | |
1871 | i += cell_length; |
1872 | } |
1873 | } |
1874 | |
1875 | #endif // QT_CONFIG(library) |
1876 | |
1877 | static void thaiAttributes(QChar::Script script, const char16_t *text, qsizetype from, qsizetype len, QCharAttributes *attributes) |
1878 | { |
1879 | assert(script == QChar::Script_Thai); |
1880 | #if QT_CONFIG(library) |
1881 | const char16_t *uc = text + from; |
1882 | attributes += from; |
1883 | Q_UNUSED(script); |
1884 | thaiAssignAttributes(string: uc, len, attributes); |
1885 | #else |
1886 | Q_UNUSED(script); |
1887 | Q_UNUSED(text); |
1888 | Q_UNUSED(from); |
1889 | Q_UNUSED(len); |
1890 | Q_UNUSED(attributes); |
1891 | #endif |
1892 | } |
1893 | |
1894 | /* |
1895 | tibetan syllables are of the form: |
1896 | head position consonant |
1897 | first sub-joined consonant |
1898 | ....intermediate sub-joined consonants (if any) |
1899 | last sub-joined consonant |
1900 | sub-joined vowel (a-chung U+0F71) |
1901 | standard or compound vowel sign (or 'virama' for devanagari transliteration) |
1902 | */ |
1903 | |
1904 | typedef enum { |
1905 | TibetanOther, |
1906 | TibetanHeadConsonant, |
1907 | TibetanSubjoinedConsonant, |
1908 | TibetanSubjoinedVowel, |
1909 | TibetanVowel |
1910 | } TibetanForm; |
1911 | |
1912 | /* this table starts at U+0f40 */ |
1913 | static const unsigned char tibetanForm[0x80] = { |
1914 | TibetanHeadConsonant, TibetanHeadConsonant, TibetanHeadConsonant, TibetanHeadConsonant, |
1915 | TibetanHeadConsonant, TibetanHeadConsonant, TibetanHeadConsonant, TibetanHeadConsonant, |
1916 | TibetanHeadConsonant, TibetanHeadConsonant, TibetanHeadConsonant, TibetanHeadConsonant, |
1917 | TibetanHeadConsonant, TibetanHeadConsonant, TibetanHeadConsonant, TibetanHeadConsonant, |
1918 | |
1919 | TibetanHeadConsonant, TibetanHeadConsonant, TibetanHeadConsonant, TibetanHeadConsonant, |
1920 | TibetanHeadConsonant, TibetanHeadConsonant, TibetanHeadConsonant, TibetanHeadConsonant, |
1921 | TibetanHeadConsonant, TibetanHeadConsonant, TibetanHeadConsonant, TibetanHeadConsonant, |
1922 | TibetanHeadConsonant, TibetanHeadConsonant, TibetanHeadConsonant, TibetanHeadConsonant, |
1923 | |
1924 | TibetanHeadConsonant, TibetanHeadConsonant, TibetanHeadConsonant, TibetanHeadConsonant, |
1925 | TibetanHeadConsonant, TibetanHeadConsonant, TibetanHeadConsonant, TibetanHeadConsonant, |
1926 | TibetanHeadConsonant, TibetanHeadConsonant, TibetanHeadConsonant, TibetanHeadConsonant, |
1927 | TibetanOther, TibetanOther, TibetanOther, TibetanOther, |
1928 | |
1929 | TibetanOther, TibetanVowel, TibetanVowel, TibetanVowel, |
1930 | TibetanVowel, TibetanVowel, TibetanVowel, TibetanVowel, |
1931 | TibetanVowel, TibetanVowel, TibetanVowel, TibetanVowel, |
1932 | TibetanVowel, TibetanVowel, TibetanVowel, TibetanVowel, |
1933 | |
1934 | TibetanVowel, TibetanVowel, TibetanVowel, TibetanVowel, |
1935 | TibetanVowel, TibetanVowel, TibetanVowel, TibetanVowel, |
1936 | TibetanOther, TibetanOther, TibetanOther, TibetanOther, |
1937 | TibetanOther, TibetanOther, TibetanOther, TibetanOther, |
1938 | |
1939 | TibetanSubjoinedConsonant, TibetanSubjoinedConsonant, TibetanSubjoinedConsonant, TibetanSubjoinedConsonant, |
1940 | TibetanSubjoinedConsonant, TibetanSubjoinedConsonant, TibetanSubjoinedConsonant, TibetanSubjoinedConsonant, |
1941 | TibetanSubjoinedConsonant, TibetanSubjoinedConsonant, TibetanSubjoinedConsonant, TibetanSubjoinedConsonant, |
1942 | TibetanSubjoinedConsonant, TibetanSubjoinedConsonant, TibetanSubjoinedConsonant, TibetanSubjoinedConsonant, |
1943 | |
1944 | TibetanSubjoinedConsonant, TibetanSubjoinedConsonant, TibetanSubjoinedConsonant, TibetanSubjoinedConsonant, |
1945 | TibetanSubjoinedConsonant, TibetanSubjoinedConsonant, TibetanSubjoinedConsonant, TibetanSubjoinedConsonant, |
1946 | TibetanSubjoinedConsonant, TibetanSubjoinedConsonant, TibetanSubjoinedConsonant, TibetanSubjoinedConsonant, |
1947 | TibetanSubjoinedConsonant, TibetanSubjoinedConsonant, TibetanSubjoinedConsonant, TibetanSubjoinedConsonant, |
1948 | |
1949 | TibetanSubjoinedConsonant, TibetanSubjoinedConsonant, TibetanSubjoinedConsonant, TibetanSubjoinedConsonant, |
1950 | TibetanSubjoinedConsonant, TibetanSubjoinedConsonant, TibetanSubjoinedConsonant, TibetanSubjoinedConsonant, |
1951 | TibetanSubjoinedConsonant, TibetanSubjoinedConsonant, TibetanSubjoinedConsonant, TibetanSubjoinedConsonant, |
1952 | TibetanSubjoinedConsonant, TibetanOther, TibetanOther, TibetanOther |
1953 | }; |
1954 | |
1955 | #define tibetan_form(c) \ |
1956 | ((c) >= 0x0f40 && (c) < 0x0fc0 ? (TibetanForm)tibetanForm[(c) - 0x0f40] : TibetanOther) |
1957 | |
1958 | static qsizetype tibetan_nextSyllableBoundary(const char16_t *s, qsizetype start, qsizetype end, bool *invalid) |
1959 | { |
1960 | const char16_t *uc = s + start; |
1961 | |
1962 | qsizetype pos = 0; |
1963 | TibetanForm state = tibetan_form(*uc); |
1964 | |
1965 | /* qDebug("state[%d]=%d (uc=%4x)", pos, state, uc[pos]);*/ |
1966 | pos++; |
1967 | |
1968 | if (state != TibetanHeadConsonant) { |
1969 | if (state != TibetanOther) |
1970 | *invalid = true; |
1971 | goto finish; |
1972 | } |
1973 | |
1974 | while (pos < end - start) { |
1975 | TibetanForm newState = tibetan_form(uc[pos]); |
1976 | switch (newState) { |
1977 | case TibetanSubjoinedConsonant: |
1978 | case TibetanSubjoinedVowel: |
1979 | if (state != TibetanHeadConsonant && |
1980 | state != TibetanSubjoinedConsonant) |
1981 | goto finish; |
1982 | state = newState; |
1983 | break; |
1984 | case TibetanVowel: |
1985 | if (state != TibetanHeadConsonant && |
1986 | state != TibetanSubjoinedConsonant && |
1987 | state != TibetanSubjoinedVowel) |
1988 | goto finish; |
1989 | break; |
1990 | case TibetanOther: |
1991 | case TibetanHeadConsonant: |
1992 | goto finish; |
1993 | } |
1994 | pos++; |
1995 | } |
1996 | |
1997 | finish: |
1998 | *invalid = false; |
1999 | return start+pos; |
2000 | } |
2001 | |
2002 | static void tibetanAttributes(QChar::Script script, const char16_t *text, qsizetype from, qsizetype len, QCharAttributes *attributes) |
2003 | { |
2004 | qsizetype end = from + len; |
2005 | qsizetype i = 0; |
2006 | Q_UNUSED(script); |
2007 | attributes += from; |
2008 | while (i < len) { |
2009 | bool invalid; |
2010 | qsizetype boundary = tibetan_nextSyllableBoundary(s: text, start: from+i, end, invalid: &invalid) - from; |
2011 | |
2012 | attributes[i].graphemeBoundary = true; |
2013 | |
2014 | if (boundary > len-1) boundary = len; |
2015 | i++; |
2016 | while (i < boundary) { |
2017 | attributes[i].graphemeBoundary = false; |
2018 | ++i; |
2019 | } |
2020 | assert(i == boundary); |
2021 | } |
2022 | } |
2023 | |
2024 | enum MymrCharClassValues { |
2025 | Mymr_CC_RESERVED = 0, |
2026 | Mymr_CC_CONSONANT = 1, /* Consonant of type 1, that has subscript form */ |
2027 | Mymr_CC_CONSONANT2 = 2, /* Consonant of type 2, that has no subscript form */ |
2028 | Mymr_CC_NGA = 3, /* Consonant NGA */ |
2029 | Mymr_CC_YA = 4, /* Consonant YA */ |
2030 | Mymr_CC_RA = 5, /* Consonant RA */ |
2031 | Mymr_CC_WA = 6, /* Consonant WA */ |
2032 | Mymr_CC_HA = 7, /* Consonant HA */ |
2033 | Mymr_CC_IND_VOWEL = 8, /* Independent vowel */ |
2034 | Mymr_CC_ZERO_WIDTH_NJ_MARK = 9, /* Zero Width non joiner character (0x200C) */ |
2035 | Mymr_CC_VIRAMA = 10, /* Subscript consonant combining character */ |
2036 | Mymr_CC_PRE_VOWEL = 11, /* Dependent vowel, prebase (Vowel e) */ |
2037 | Mymr_CC_BELOW_VOWEL = 12, /* Dependent vowel, prebase (Vowel u, uu) */ |
2038 | Mymr_CC_ABOVE_VOWEL = 13, /* Dependent vowel, prebase (Vowel i, ii, ai) */ |
2039 | Mymr_CC_POST_VOWEL = 14, /* Dependent vowel, prebase (Vowel aa) */ |
2040 | Mymr_CC_SIGN_ABOVE = 15, |
2041 | Mymr_CC_SIGN_BELOW = 16, |
2042 | Mymr_CC_SIGN_AFTER = 17, |
2043 | Mymr_CC_ZERO_WIDTH_J_MARK = 18, /* Zero width joiner character */ |
2044 | Mymr_CC_COUNT = 19 /* This is the number of character classes */ |
2045 | }; |
2046 | |
2047 | enum MymrCharClassFlags { |
2048 | Mymr_CF_CLASS_MASK = 0x0000FFFF, |
2049 | |
2050 | Mymr_CF_CONSONANT = 0x01000000, /* flag to speed up comparing */ |
2051 | Mymr_CF_MEDIAL = 0x02000000, /* flag to speed up comparing */ |
2052 | Mymr_CF_IND_VOWEL = 0x04000000, /* flag to speed up comparing */ |
2053 | Mymr_CF_DEP_VOWEL = 0x08000000, /* flag to speed up comparing */ |
2054 | Mymr_CF_DOTTED_CIRCLE = 0x10000000, /* add a dotted circle if a character with this flag is the |
2055 | first in a syllable */ |
2056 | Mymr_CF_VIRAMA = 0x20000000, /* flag to speed up comparing */ |
2057 | |
2058 | /* position flags */ |
2059 | Mymr_CF_POS_BEFORE = 0x00080000, |
2060 | Mymr_CF_POS_BELOW = 0x00040000, |
2061 | Mymr_CF_POS_ABOVE = 0x00020000, |
2062 | Mymr_CF_POS_AFTER = 0x00010000, |
2063 | Mymr_CF_POS_MASK = 0x000f0000, |
2064 | |
2065 | Mymr_CF_AFTER_KINZI = 0x00100000 |
2066 | }; |
2067 | |
2068 | Q_DECLARE_MIXED_ENUM_OPERATORS(int, MymrCharClassValues, MymrCharClassFlags) |
2069 | |
2070 | /* Characters that get refrered to by name */ |
2071 | enum MymrChar |
2072 | { |
2073 | Mymr_C_SIGN_ZWNJ = 0x200C, |
2074 | Mymr_C_SIGN_ZWJ = 0x200D, |
2075 | Mymr_C_DOTTED_CIRCLE = 0x25CC, |
2076 | Mymr_C_RA = 0x101B, |
2077 | Mymr_C_YA = 0x101A, |
2078 | Mymr_C_NGA = 0x1004, |
2079 | Mymr_C_VOWEL_E = 0x1031, |
2080 | Mymr_C_VIRAMA = 0x1039 |
2081 | }; |
2082 | |
2083 | enum |
2084 | { |
2085 | Mymr_xx = Mymr_CC_RESERVED, |
2086 | Mymr_c1 = Mymr_CC_CONSONANT | Mymr_CF_CONSONANT | Mymr_CF_POS_BELOW, |
2087 | Mymr_c2 = Mymr_CC_CONSONANT2 | Mymr_CF_CONSONANT, |
2088 | Mymr_ng = Mymr_CC_NGA | Mymr_CF_CONSONANT | Mymr_CF_POS_ABOVE, |
2089 | Mymr_ya = Mymr_CC_YA | Mymr_CF_CONSONANT | Mymr_CF_MEDIAL | Mymr_CF_POS_AFTER | Mymr_CF_AFTER_KINZI, |
2090 | Mymr_ra = Mymr_CC_RA | Mymr_CF_CONSONANT | Mymr_CF_MEDIAL | Mymr_CF_POS_BEFORE, |
2091 | Mymr_wa = Mymr_CC_WA | Mymr_CF_CONSONANT | Mymr_CF_MEDIAL | Mymr_CF_POS_BELOW, |
2092 | Mymr_ha = Mymr_CC_HA | Mymr_CF_CONSONANT | Mymr_CF_MEDIAL | Mymr_CF_POS_BELOW, |
2093 | Mymr_id = Mymr_CC_IND_VOWEL | Mymr_CF_IND_VOWEL, |
2094 | Mymr_vi = Mymr_CC_VIRAMA | Mymr_CF_VIRAMA | Mymr_CF_POS_ABOVE | Mymr_CF_DOTTED_CIRCLE, |
2095 | Mymr_dl = Mymr_CC_PRE_VOWEL | Mymr_CF_DEP_VOWEL | Mymr_CF_POS_BEFORE | Mymr_CF_DOTTED_CIRCLE | Mymr_CF_AFTER_KINZI, |
2096 | Mymr_db = Mymr_CC_BELOW_VOWEL | Mymr_CF_DEP_VOWEL | Mymr_CF_POS_BELOW | Mymr_CF_DOTTED_CIRCLE | Mymr_CF_AFTER_KINZI, |
2097 | Mymr_da = Mymr_CC_ABOVE_VOWEL | Mymr_CF_DEP_VOWEL | Mymr_CF_POS_ABOVE | Mymr_CF_DOTTED_CIRCLE | Mymr_CF_AFTER_KINZI, |
2098 | Mymr_dr = Mymr_CC_POST_VOWEL | Mymr_CF_DEP_VOWEL | Mymr_CF_POS_AFTER | Mymr_CF_DOTTED_CIRCLE | Mymr_CF_AFTER_KINZI, |
2099 | Mymr_sa = Mymr_CC_SIGN_ABOVE | Mymr_CF_DOTTED_CIRCLE | Mymr_CF_POS_ABOVE | Mymr_CF_AFTER_KINZI, |
2100 | Mymr_sb = Mymr_CC_SIGN_BELOW | Mymr_CF_DOTTED_CIRCLE | Mymr_CF_POS_BELOW | Mymr_CF_AFTER_KINZI, |
2101 | Mymr_sp = Mymr_CC_SIGN_AFTER | Mymr_CF_DOTTED_CIRCLE | Mymr_CF_AFTER_KINZI |
2102 | }; |
2103 | |
2104 | |
2105 | typedef int MymrCharClass; |
2106 | |
2107 | |
2108 | static const MymrCharClass mymrCharClasses[] = |
2109 | { |
2110 | Mymr_c1, Mymr_c1, Mymr_c1, Mymr_c1, Mymr_ng, Mymr_c1, Mymr_c1, Mymr_c1, |
2111 | Mymr_c1, Mymr_c1, Mymr_c2, Mymr_c1, Mymr_c1, Mymr_c1, Mymr_c1, Mymr_c1, /* 1000 - 100F */ |
2112 | Mymr_c1, Mymr_c1, Mymr_c1, Mymr_c1, Mymr_c1, Mymr_c1, Mymr_c1, Mymr_c1, |
2113 | Mymr_c1, Mymr_c1, Mymr_ya, Mymr_ra, Mymr_c1, Mymr_wa, Mymr_c1, Mymr_ha, /* 1010 - 101F */ |
2114 | Mymr_c2, Mymr_c2, Mymr_xx, Mymr_id, Mymr_id, Mymr_id, Mymr_id, Mymr_id, |
2115 | Mymr_xx, Mymr_id, Mymr_id, Mymr_xx, Mymr_dr, Mymr_da, Mymr_da, Mymr_db, /* 1020 - 102F */ |
2116 | Mymr_db, Mymr_dl, Mymr_da, Mymr_xx, Mymr_xx, Mymr_xx, Mymr_sa, Mymr_sb, |
2117 | Mymr_sp, Mymr_vi, Mymr_xx, Mymr_xx, Mymr_xx, Mymr_xx, Mymr_xx, Mymr_xx, /* 1030 - 103F */ |
2118 | Mymr_xx, Mymr_xx, Mymr_xx, Mymr_xx, Mymr_xx, Mymr_xx, Mymr_xx, Mymr_xx, |
2119 | Mymr_xx, Mymr_xx, Mymr_xx, Mymr_xx, Mymr_xx, Mymr_xx, Mymr_xx, Mymr_xx, /* 1040 - 104F */ |
2120 | Mymr_xx, Mymr_xx, Mymr_xx, Mymr_xx, Mymr_xx, Mymr_xx, Mymr_xx, Mymr_xx, |
2121 | Mymr_xx, Mymr_xx, Mymr_xx, Mymr_xx, Mymr_xx, Mymr_xx, Mymr_xx, Mymr_xx, /* 1050 - 105F */ |
2122 | }; |
2123 | |
2124 | static MymrCharClass |
2125 | getMyanmarCharClass (ushort ch) |
2126 | { |
2127 | if (ch == Mymr_C_SIGN_ZWJ) |
2128 | return Mymr_CC_ZERO_WIDTH_J_MARK; |
2129 | |
2130 | if (ch == Mymr_C_SIGN_ZWNJ) |
2131 | return Mymr_CC_ZERO_WIDTH_NJ_MARK; |
2132 | |
2133 | if (ch < 0x1000 || ch > 0x105f) |
2134 | return Mymr_CC_RESERVED; |
2135 | |
2136 | return mymrCharClasses[ch - 0x1000]; |
2137 | } |
2138 | |
2139 | static const signed char mymrStateTable[][Mymr_CC_COUNT] = |
2140 | { |
2141 | /* xx c1, c2 ng ya ra wa ha id zwnj vi dl db da dr sa sb sp zwj */ |
2142 | { 1, 4, 4, 2, 4, 4, 4, 4, 24, 1, 27, 17, 18, 19, 20, 21, 1, 1, 4}, /* 0 - ground state */ |
2143 | {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}, /* 1 - exit state (or sp to the right of the syllable) */ |
2144 | {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 3, 17, 18, 19, 20, 21, -1, -1, 4}, /* 2 - NGA */ |
2145 | {-1, 4, 4, 4, 4, 4, 4, 4, -1, 23, -1, -1, -1, -1, -1, -1, -1, -1, -1}, /* 3 - Virama after NGA */ |
2146 | {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 5, 17, 18, 19, 20, 21, 1, 1, -1}, /* 4 - Base consonant */ |
2147 | {-2, 6, -2, -2, 7, 8, 9, 10, -2, 23, -2, -2, -2, -2, -2, -2, -2, -2, -2}, /* 5 - First virama */ |
2148 | {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 25, 17, 18, 19, 20, 21, -1, -1, -1}, /* 6 - c1 after virama */ |
2149 | {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 12, 17, 18, 19, 20, 21, -1, -1, -1}, /* 7 - ya after virama */ |
2150 | {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 12, 17, 18, 19, 20, 21, -1, -1, -1}, /* 8 - ra after virama */ |
2151 | {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 12, 17, 18, 19, 20, 21, -1, -1, -1}, /* 9 - wa after virama */ |
2152 | {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 17, 18, 19, 20, 21, -1, -1, -1}, /* 10 - ha after virama */ |
2153 | {-1, -1, -1, -1, 7, 8, 9, 10, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}, /* 11 - Virama after NGA+zwj */ |
2154 | {-2, -2, -2, -2, -2, -2, 13, 14, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2}, /* 12 - Second virama */ |
2155 | {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 15, 17, 18, 19, 20, 21, -1, -1, -1}, /* 13 - wa after virama */ |
2156 | {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 17, 18, 19, 20, 21, -1, -1, -1}, /* 14 - ha after virama */ |
2157 | {-2, -2, -2, -2, -2, -2, -2, 16, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2}, /* 15 - Third virama */ |
2158 | {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 17, 18, 19, 20, 21, -1, -1, -1}, /* 16 - ha after virama */ |
2159 | {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 20, 21, 1, 1, -1}, /* 17 - dl, Dependent vowel e */ |
2160 | {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 19, -1, 21, 1, 1, -1}, /* 18 - db, Dependent vowel u,uu */ |
2161 | {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, 1, 1, -1}, /* 19 - da, Dependent vowel i,ii,ai */ |
2162 | {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 22, -1, -1, -1, -1, -1, 1, 1, -1}, /* 20 - dr, Dependent vowel aa */ |
2163 | {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, 1, -1}, /* 21 - sa, Sign anusvara */ |
2164 | {-1, -1, -1, -1, -1, -1, -1, -1, -1, 23, -1, -1, -1, -1, -1, -1, -1, -1, -1}, /* 22 - atha */ |
2165 | {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, 1, -1}, /* 23 - zwnj for atha */ |
2166 | {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, -1}, /* 24 - Independent vowel */ |
2167 | {-2, -2, -2, -2, 26, 26, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2}, /* 25 - Virama after subscript consonant */ |
2168 | {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 12, 17, 18, 19, 20, 21, -1, 1, -1}, /* 26 - ra/ya after subscript consonant + virama */ |
2169 | {-1, 6, -1, -1, 7, 8, 9, 10, -1, 23, -1, -1, -1, -1, -1, -1, -1, -1, -1}, /* 27 - Virama after ground state */ |
2170 | /* exit state -2 is for invalid order of medials and combination of invalids |
2171 | with virama where virama should treat as start of next syllable |
2172 | */ |
2173 | }; |
2174 | |
2175 | /*#define MYANMAR_DEBUG */ |
2176 | #ifdef MYANMAR_DEBUG |
2177 | #define MMDEBUG qDebug |
2178 | #else |
2179 | # define MMDEBUG \ |
2180 | if (0) \ |
2181 | printf |
2182 | #endif |
2183 | |
2184 | /* |
2185 | // Given an input string of characters and a location in which to start looking |
2186 | // calculate, using the state table, which one is the last character of the syllable |
2187 | // that starts in the starting position. |
2188 | */ |
2189 | static qsizetype myanmar_nextSyllableBoundary(const char16_t *s, qsizetype start, qsizetype end, bool *invalid) |
2190 | { |
2191 | const char16_t *uc = s + start; |
2192 | int state = 0; |
2193 | qsizetype pos = start; |
2194 | *invalid = false; |
2195 | |
2196 | while (pos < end) { |
2197 | MymrCharClass charClass = getMyanmarCharClass(ch: *uc); |
2198 | state = mymrStateTable[state][charClass & Mymr_CF_CLASS_MASK]; |
2199 | if (pos == start) |
2200 | *invalid = (bool)(charClass & Mymr_CF_DOTTED_CIRCLE); |
2201 | |
2202 | MMDEBUG(format: "state[%lld]=%d class=%8x (uc=%4x)", qlonglong(pos - start), state, charClass, *uc); |
2203 | |
2204 | if (state < 0) { |
2205 | if (state < -1) |
2206 | --pos; |
2207 | break; |
2208 | } |
2209 | ++uc; |
2210 | ++pos; |
2211 | } |
2212 | return pos; |
2213 | } |
2214 | |
2215 | static void myanmarAttributes(QChar::Script script, const char16_t *text, qsizetype from, qsizetype len, QCharAttributes *attributes) |
2216 | { |
2217 | qsizetype end = from + len; |
2218 | qsizetype i = 0; |
2219 | Q_UNUSED(script); |
2220 | attributes += from; |
2221 | while (i < len) { |
2222 | bool invalid; |
2223 | qsizetype boundary = myanmar_nextSyllableBoundary(s: text, start: from+i, end, invalid: &invalid) - from; |
2224 | |
2225 | attributes[i].graphemeBoundary = true; |
2226 | attributes[i].lineBreak = true; |
2227 | |
2228 | if (boundary > len-1) |
2229 | boundary = len; |
2230 | i++; |
2231 | while (i < boundary) { |
2232 | attributes[i].graphemeBoundary = false; |
2233 | ++i; |
2234 | } |
2235 | assert(i == boundary); |
2236 | } |
2237 | } |
2238 | |
2239 | /* |
2240 | // Vocabulary |
2241 | // Base -> A consonant or an independent vowel in its full (not subscript) form. It is the |
2242 | // center of the syllable, it can be surrounded by coeng (subscript) consonants, vowels, |
2243 | // split vowels, signs... but there is only one base in a syllable, it has to be coded as |
2244 | // the first character of the syllable. |
2245 | // split vowel --> vowel that has two parts placed separately (e.g. Before and after the consonant). |
2246 | // Khmer language has five of them. Khmer split vowels either have one part before the |
2247 | // base and one after the base or they have a part before the base and a part above the base. |
2248 | // The first part of all Khmer split vowels is the same character, identical to |
2249 | // the glyph of Khmer dependent vowel SRA EI |
2250 | // coeng --> modifier used in Khmer to construct coeng (subscript) consonants |
2251 | // Differently than indian languages, the coeng modifies the consonant that follows it, |
2252 | // not the one preceding it Each consonant has two forms, the base form and the subscript form |
2253 | // the base form is the normal one (using the consonants code-point), the subscript form is |
2254 | // displayed when the combination coeng + consonant is encountered. |
2255 | // Consonant of type 1 -> A consonant which has subscript for that only occupies space under a base consonant |
2256 | // Consonant of type 2.-> Its subscript form occupies space under and before the base (only one, RO) |
2257 | // Consonant of Type 3 -> Its subscript form occupies space under and after the base (KHO, CHHO, THHO, BA, YO, SA) |
2258 | // Consonant shifter -> Khmer has to series of consonants. The same dependent vowel has different sounds |
2259 | // if it is attached to a consonant of the first series or a consonant of the second series |
2260 | // Most consonants have an equivalent in the other series, but some of theme exist only in |
2261 | // one series (for example SA). If we want to use the consonant SA with a vowel sound that |
2262 | // can only be done with a vowel sound that corresponds to a vowel accompanying a consonant |
2263 | // of the other series, then we need to use a consonant shifter: TRIISAP or MUSIKATOAN |
2264 | // x17C9 y x17CA. TRIISAP changes a first series consonant to second series sound and |
2265 | // MUSIKATOAN a second series consonant to have a first series vowel sound. |
2266 | // Consonant shifter are both normally supercript marks, but, when they are followed by a |
2267 | // superscript, they change shape and take the form of subscript dependent vowel SRA U. |
2268 | // If they are in the same syllable as a coeng consonant, Unicode 3.0 says that they |
2269 | // should be typed before the coeng. Unicode 4.0 breaks the standard and says that it should |
2270 | // be placed after the coeng consonant. |
2271 | // Dependent vowel -> In khmer dependent vowels can be placed above, below, before or after the base |
2272 | // Each vowel has its own position. Only one vowel per syllable is allowed. |
2273 | // Signs -> Khmer has above signs and post signs. Only one above sign and/or one post sign are |
2274 | // Allowed in a syllable. |
2275 | // |
2276 | // |
2277 | // order is important here! This order must be the same that is found in each horizontal |
2278 | // line in the statetable for Khmer (see khmerStateTable) . |
2279 | */ |
2280 | enum KhmerCharClassValues { |
2281 | CC_RESERVED = 0, |
2282 | CC_CONSONANT = 1, /* Consonant of type 1 or independent vowel */ |
2283 | CC_CONSONANT2 = 2, /* Consonant of type 2 */ |
2284 | CC_CONSONANT3 = 3, /* Consonant of type 3 */ |
2285 | CC_ZERO_WIDTH_NJ_MARK = 4, /* Zero Width non joiner character (0x200C) */ |
2286 | CC_CONSONANT_SHIFTER = 5, |
2287 | CC_ROBAT = 6, /* Khmer special diacritic accent -treated differently in state table */ |
2288 | CC_COENG = 7, /* Subscript consonant combining character */ |
2289 | CC_DEPENDENT_VOWEL = 8, |
2290 | CC_SIGN_ABOVE = 9, |
2291 | CC_SIGN_AFTER = 10, |
2292 | CC_ZERO_WIDTH_J_MARK = 11, /* Zero width joiner character */ |
2293 | CC_COUNT = 12 /* This is the number of character classes */ |
2294 | }; |
2295 | |
2296 | |
2297 | enum KhmerCharClassFlags { |
2298 | CF_CLASS_MASK = 0x0000FFFF, |
2299 | |
2300 | CF_CONSONANT = 0x01000000, /* flag to speed up comparing */ |
2301 | CF_SPLIT_VOWEL = 0x02000000, /* flag for a split vowel -> the first part is added in front of the syllable */ |
2302 | CF_DOTTED_CIRCLE = 0x04000000, /* add a dotted circle if a character with this flag is the first in a syllable */ |
2303 | CF_COENG = 0x08000000, /* flag to speed up comparing */ |
2304 | CF_SHIFTER = 0x10000000, /* flag to speed up comparing */ |
2305 | CF_ABOVE_VOWEL = 0x20000000, /* flag to speed up comparing */ |
2306 | |
2307 | /* position flags */ |
2308 | CF_POS_BEFORE = 0x00080000, |
2309 | CF_POS_BELOW = 0x00040000, |
2310 | CF_POS_ABOVE = 0x00020000, |
2311 | CF_POS_AFTER = 0x00010000, |
2312 | CF_POS_MASK = 0x000f0000 |
2313 | }; |
2314 | |
2315 | Q_DECLARE_MIXED_ENUM_OPERATORS(int, KhmerCharClassValues, KhmerCharClassFlags) |
2316 | |
2317 | /* Characters that get referred to by name */ |
2318 | enum KhmerChar { |
2319 | C_SIGN_ZWNJ = 0x200C, |
2320 | C_SIGN_ZWJ = 0x200D, |
2321 | C_RO = 0x179A, |
2322 | C_VOWEL_AA = 0x17B6, |
2323 | C_SIGN_NIKAHIT = 0x17C6, |
2324 | C_VOWEL_E = 0x17C1, |
2325 | C_COENG = 0x17D2 |
2326 | }; |
2327 | |
2328 | |
2329 | /* |
2330 | // simple classes, they are used in the statetable (in this file) to control the length of a syllable |
2331 | // they are also used to know where a character should be placed (location in reference to the base character) |
2332 | // and also to know if a character, when independently displayed, should be displayed with a dotted-circle to |
2333 | // indicate error in syllable construction |
2334 | */ |
2335 | enum { |
2336 | _xx = CC_RESERVED, |
2337 | _sa = CC_SIGN_ABOVE | CF_DOTTED_CIRCLE | CF_POS_ABOVE, |
2338 | _sp = CC_SIGN_AFTER | CF_DOTTED_CIRCLE| CF_POS_AFTER, |
2339 | _c1 = CC_CONSONANT | CF_CONSONANT, |
2340 | _c2 = CC_CONSONANT2 | CF_CONSONANT, |
2341 | _c3 = CC_CONSONANT3 | CF_CONSONANT, |
2342 | _rb = CC_ROBAT | CF_POS_ABOVE | CF_DOTTED_CIRCLE, |
2343 | _cs = CC_CONSONANT_SHIFTER | CF_DOTTED_CIRCLE | CF_SHIFTER, |
2344 | _dl = CC_DEPENDENT_VOWEL | CF_POS_BEFORE | CF_DOTTED_CIRCLE, |
2345 | _db = CC_DEPENDENT_VOWEL | CF_POS_BELOW | CF_DOTTED_CIRCLE, |
2346 | _da = CC_DEPENDENT_VOWEL | CF_POS_ABOVE | CF_DOTTED_CIRCLE | CF_ABOVE_VOWEL, |
2347 | _dr = CC_DEPENDENT_VOWEL | CF_POS_AFTER | CF_DOTTED_CIRCLE, |
2348 | _co = CC_COENG | CF_COENG | CF_DOTTED_CIRCLE, |
2349 | |
2350 | /* split vowel */ |
2351 | _va = _da | CF_SPLIT_VOWEL, |
2352 | _vr = _dr | CF_SPLIT_VOWEL |
2353 | }; |
2354 | |
2355 | |
2356 | /* |
2357 | // Character class: a character class value |
2358 | // ORed with character class flags. |
2359 | */ |
2360 | typedef unsigned long KhmerCharClass; |
2361 | |
2362 | |
2363 | /* |
2364 | // Character class tables |
2365 | // _xx character does not combine into syllable, such as numbers, puntuation marks, non-Khmer signs... |
2366 | // _sa Sign placed above the base |
2367 | // _sp Sign placed after the base |
2368 | // _c1 Consonant of type 1 or independent vowel (independent vowels behave as type 1 consonants) |
2369 | // _c2 Consonant of type 2 (only RO) |
2370 | // _c3 Consonant of type 3 |
2371 | // _rb Khmer sign robat u17CC. combining mark for subscript consonants |
2372 | // _cd Consonant-shifter |
2373 | // _dl Dependent vowel placed before the base (left of the base) |
2374 | // _db Dependent vowel placed below the base |
2375 | // _da Dependent vowel placed above the base |
2376 | // _dr Dependent vowel placed behind the base (right of the base) |
2377 | // _co Khmer combining mark COENG u17D2, combines with the consonant or independent vowel following |
2378 | // it to create a subscript consonant or independent vowel |
2379 | // _va Khmer split vowel in which the first part is before the base and the second one above the base |
2380 | // _vr Khmer split vowel in which the first part is before the base and the second one behind (right of) the base |
2381 | */ |
2382 | static const KhmerCharClass khmerCharClasses[] = { |
2383 | _c1, _c1, _c1, _c3, _c1, _c1, _c1, _c1, _c3, _c1, _c1, _c1, _c1, _c3, _c1, _c1, /* 1780 - 178F */ |
2384 | _c1, _c1, _c1, _c1, _c3, _c1, _c1, _c1, _c1, _c3, _c2, _c1, _c1, _c1, _c3, _c3, /* 1790 - 179F */ |
2385 | _c1, _c3, _c1, _c1, _c1, _c1, _c1, _c1, _c1, _c1, _c1, _c1, _c1, _c1, _c1, _c1, /* 17A0 - 17AF */ |
2386 | _c1, _c1, _c1, _c1, _dr, _dr, _dr, _da, _da, _da, _da, _db, _db, _db, _va, _vr, /* 17B0 - 17BF */ |
2387 | _vr, _dl, _dl, _dl, _vr, _vr, _sa, _sp, _sp, _cs, _cs, _sa, _rb, _sa, _sa, _sa, /* 17C0 - 17CF */ |
2388 | _sa, _sa, _co, _sa, _xx, _xx, _xx, _xx, _xx, _xx, _xx, _xx, _xx, _sa, _xx, _xx /* 17D0 - 17DF */ |
2389 | }; |
2390 | |
2391 | /* this enum must reflect the range of khmerCharClasses */ |
2392 | enum KhmerCharClassesRange { |
2393 | KhmerFirstChar = 0x1780, |
2394 | KhmerLastChar = 0x17df |
2395 | }; |
2396 | |
2397 | /* |
2398 | // Below we define how a character in the input string is either in the khmerCharClasses table |
2399 | // (in which case we get its type back), a ZWJ or ZWNJ (two characters that may appear |
2400 | // within the syllable, but are not in the table) we also get their type back, or an unknown object |
2401 | // in which case we get _xx (CC_RESERVED) back |
2402 | */ |
2403 | static KhmerCharClass getKhmerCharClass(ushort uc) |
2404 | { |
2405 | if (uc == C_SIGN_ZWJ) { |
2406 | return CC_ZERO_WIDTH_J_MARK; |
2407 | } |
2408 | |
2409 | if (uc == C_SIGN_ZWNJ) { |
2410 | return CC_ZERO_WIDTH_NJ_MARK; |
2411 | } |
2412 | |
2413 | if (uc < KhmerFirstChar || uc > KhmerLastChar) { |
2414 | return CC_RESERVED; |
2415 | } |
2416 | |
2417 | return khmerCharClasses[uc - KhmerFirstChar]; |
2418 | } |
2419 | |
2420 | |
2421 | /* |
2422 | // The stateTable is used to calculate the end (the length) of a well |
2423 | // formed Khmer Syllable. |
2424 | // |
2425 | // Each horizontal line is ordered exactly the same way as the values in KhmerClassTable |
2426 | // CharClassValues. This coincidence of values allows the follow up of the table. |
2427 | // |
2428 | // Each line corresponds to a state, which does not necessarily need to be a type |
2429 | // of component... for example, state 2 is a base, with is always a first character |
2430 | // in the syllable, but the state could be produced a consonant of any type when |
2431 | // it is the first character that is analysed (in ground state). |
2432 | // |
2433 | // Differentiating 3 types of consonants is necessary in order to |
2434 | // forbid the use of certain combinations, such as having a second |
2435 | // coeng after a coeng RO, |
2436 | // The inexistent possibility of having a type 3 after another type 3 is permitted, |
2437 | // eliminating it would very much complicate the table, and it does not create typing |
2438 | // problems, as the case above. |
2439 | // |
2440 | // The table is quite complex, in order to limit the number of coeng consonants |
2441 | // to 2 (by means of the table). |
2442 | // |
2443 | // There a peculiarity, as far as Unicode is concerned: |
2444 | // - The consonant-shifter is considered in two possible different |
2445 | // locations, the one considered in Unicode 3.0 and the one considered in |
2446 | // Unicode 4.0. (there is a backwards compatibility problem in this standard). |
2447 | // |
2448 | // |
2449 | // xx independent character, such as a number, punctuation sign or non-khmer char |
2450 | // |
2451 | // c1 Khmer consonant of type 1 or an independent vowel |
2452 | // that is, a letter in which the subscript for is only under the |
2453 | // base, not taking any space to the right or to the left |
2454 | // |
2455 | // c2 Khmer consonant of type 2, the coeng form takes space under |
2456 | // and to the left of the base (only RO is of this type) |
2457 | // |
2458 | // c3 Khmer consonant of type 3. Its subscript form takes space under |
2459 | // and to the right of the base. |
2460 | // |
2461 | // cs Khmer consonant shifter |
2462 | // |
2463 | // rb Khmer robat |
2464 | // |
2465 | // co coeng character (u17D2) |
2466 | // |
2467 | // dv dependent vowel (including split vowels, they are treated in the same way). |
2468 | // even if dv is not defined above, the component that is really tested for is |
2469 | // KhmerClassTable::CC_DEPENDENT_VOWEL, which is common to all dependent vowels |
2470 | // |
2471 | // zwj Zero Width joiner |
2472 | // |
2473 | // zwnj Zero width non joiner |
2474 | // |
2475 | // sa above sign |
2476 | // |
2477 | // sp post sign |
2478 | // |
2479 | // there are lines with equal content but for an easier understanding |
2480 | // (and maybe change in the future) we did not join them |
2481 | */ |
2482 | static const signed char khmerStateTable[][CC_COUNT] = |
2483 | { |
2484 | /* xx c1 c2 c3 zwnj cs rb co dv sa sp zwj */ |
2485 | { 1, 2, 2, 2, 1, 1, 1, 6, 1, 1, 1, 2}, /* 0 - ground state */ |
2486 | {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}, /* 1 - exit state (or sign to the right of the syllable) */ |
2487 | {-1, -1, -1, -1, 3, 4, 5, 6, 16, 17, 1, -1}, /* 2 - Base consonant */ |
2488 | {-1, -1, -1, -1, -1, 4, -1, -1, 16, -1, -1, -1}, /* 3 - First ZWNJ before a register shifter It can only be followed by a shifter or a vowel */ |
2489 | {-1, -1, -1, -1, 15, -1, -1, 6, 16, 17, 1, 14}, /* 4 - First register shifter */ |
2490 | {-1, -1, -1, -1, -1, -1, -1, -1, 20, -1, 1, -1}, /* 5 - Robat */ |
2491 | {-1, 7, 8, 9, -1, -1, -1, -1, -1, -1, -1, -1}, /* 6 - First Coeng */ |
2492 | {-1, -1, -1, -1, 12, 13, -1, 10, 16, 17, 1, 14}, /* 7 - First consonant of type 1 after coeng */ |
2493 | {-1, -1, -1, -1, 12, 13, -1, -1, 16, 17, 1, 14}, /* 8 - First consonant of type 2 after coeng */ |
2494 | {-1, -1, -1, -1, 12, 13, -1, 10, 16, 17, 1, 14}, /* 9 - First consonant or type 3 after ceong */ |
2495 | {-1, 11, 11, 11, -1, -1, -1, -1, -1, -1, -1, -1}, /* 10 - Second Coeng (no register shifter before) */ |
2496 | {-1, -1, -1, -1, 15, -1, -1, -1, 16, 17, 1, 14}, /* 11 - Second coeng consonant (or ind. vowel) no register shifter before */ |
2497 | {-1, -1, -1, -1, -1, 13, -1, -1, 16, -1, -1, -1}, /* 12 - Second ZWNJ before a register shifter */ |
2498 | {-1, -1, -1, -1, 15, -1, -1, -1, 16, 17, 1, 14}, /* 13 - Second register shifter */ |
2499 | {-1, -1, -1, -1, -1, -1, -1, -1, 16, -1, -1, -1}, /* 14 - ZWJ before vowel */ |
2500 | {-1, -1, -1, -1, -1, -1, -1, -1, 16, -1, -1, -1}, /* 15 - ZWNJ before vowel */ |
2501 | {-1, -1, -1, -1, -1, -1, -1, -1, -1, 17, 1, 18}, /* 16 - dependent vowel */ |
2502 | {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, 18}, /* 17 - sign above */ |
2503 | {-1, -1, -1, -1, -1, -1, -1, 19, -1, -1, -1, -1}, /* 18 - ZWJ after vowel */ |
2504 | {-1, 1, -1, 1, -1, -1, -1, -1, -1, -1, -1, -1}, /* 19 - Third coeng */ |
2505 | {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, -1}, /* 20 - dependent vowel after a Robat */ |
2506 | }; |
2507 | |
2508 | |
2509 | /* #define KHMER_DEBUG */ |
2510 | #ifdef KHMER_DEBUG |
2511 | #define KHDEBUG qDebug |
2512 | #else |
2513 | # define KHDEBUG \ |
2514 | if (0) \ |
2515 | printf |
2516 | #endif |
2517 | |
2518 | /* |
2519 | // Given an input string of characters and a location in which to start looking |
2520 | // calculate, using the state table, which one is the last character of the syllable |
2521 | // that starts in the starting position. |
2522 | */ |
2523 | static qsizetype khmer_nextSyllableBoundary(const char16_t *s, qsizetype start, qsizetype end, bool *invalid) |
2524 | { |
2525 | const char16_t *uc = s + start; |
2526 | int state = 0; |
2527 | qsizetype pos = start; |
2528 | *invalid = false; |
2529 | |
2530 | while (pos < end) { |
2531 | KhmerCharClass charClass = getKhmerCharClass(uc: *uc); |
2532 | if (pos == start) { |
2533 | *invalid = (charClass > 0) && ! (charClass & CF_CONSONANT); |
2534 | } |
2535 | state = khmerStateTable[state][charClass & CF_CLASS_MASK]; |
2536 | |
2537 | KHDEBUG(format: "state[%lld]=%d class=%8lx (uc=%4x)", qlonglong(pos - start), state, |
2538 | charClass, *uc ); |
2539 | |
2540 | if (state < 0) { |
2541 | break; |
2542 | } |
2543 | ++uc; |
2544 | ++pos; |
2545 | } |
2546 | return pos; |
2547 | } |
2548 | |
2549 | static void khmerAttributes(QChar::Script script, const char16_t *text, qsizetype from, qsizetype len, QCharAttributes *attributes) |
2550 | { |
2551 | qsizetype end = from + len; |
2552 | qsizetype i = 0; |
2553 | Q_UNUSED(script); |
2554 | attributes += from; |
2555 | while ( i < len ) { |
2556 | bool invalid; |
2557 | qsizetype boundary = khmer_nextSyllableBoundary( s: text, start: from+i, end, invalid: &invalid ) - from; |
2558 | |
2559 | attributes[i].graphemeBoundary = true; |
2560 | |
2561 | if ( boundary > len-1 ) boundary = len; |
2562 | i++; |
2563 | while ( i < boundary ) { |
2564 | attributes[i].graphemeBoundary = false; |
2565 | ++i; |
2566 | } |
2567 | assert( i == boundary ); |
2568 | } |
2569 | } |
2570 | |
2571 | |
2572 | const CharAttributeFunction charAttributeFunction[] = { |
2573 | // Script_Unknown, |
2574 | nullptr, |
2575 | // Script_Inherited, |
2576 | nullptr, |
2577 | // Script_Common, |
2578 | nullptr, |
2579 | // Script_Latin, |
2580 | nullptr, |
2581 | // Script_Greek, |
2582 | nullptr, |
2583 | // Script_Cyrillic, |
2584 | nullptr, |
2585 | // Script_Armenian, |
2586 | nullptr, |
2587 | // Script_Hebrew, |
2588 | nullptr, |
2589 | // Script_Arabic, |
2590 | nullptr, |
2591 | // Script_Syriac, |
2592 | nullptr, |
2593 | // Script_Thaana, |
2594 | nullptr, |
2595 | // Script_Devanagari, |
2596 | indicAttributes, |
2597 | // Script_Bengali, |
2598 | indicAttributes, |
2599 | // Script_Gurmukhi, |
2600 | indicAttributes, |
2601 | // Script_Gujarati, |
2602 | indicAttributes, |
2603 | // Script_Oriya, |
2604 | indicAttributes, |
2605 | // Script_Tamil, |
2606 | indicAttributes, |
2607 | // Script_Telugu, |
2608 | indicAttributes, |
2609 | // Script_Kannada, |
2610 | indicAttributes, |
2611 | // Script_Malayalam, |
2612 | indicAttributes, |
2613 | // Script_Sinhala, |
2614 | indicAttributes, |
2615 | // Script_Thai, |
2616 | thaiAttributes, |
2617 | // Script_Lao, |
2618 | nullptr, |
2619 | // Script_Tibetan, |
2620 | tibetanAttributes, |
2621 | // Script_Myanmar, |
2622 | myanmarAttributes, |
2623 | // Script_Georgian, |
2624 | nullptr, |
2625 | // Script_Hangul, |
2626 | nullptr, |
2627 | // Script_Ethiopic, |
2628 | nullptr, |
2629 | // Script_Cherokee, |
2630 | nullptr, |
2631 | // Script_CanadianAboriginal, |
2632 | nullptr, |
2633 | // Script_Ogham, |
2634 | nullptr, |
2635 | // Script_Runic, |
2636 | nullptr, |
2637 | // Script_Khmer, |
2638 | khmerAttributes |
2639 | }; |
2640 | |
2641 | static void getCharAttributes(const char16_t *string, qsizetype stringLength, |
2642 | const QUnicodeTools::ScriptItem *items, qsizetype numItems, |
2643 | QCharAttributes *attributes) |
2644 | { |
2645 | if (stringLength == 0) |
2646 | return; |
2647 | for (qsizetype i = 0; i < numItems; ++i) { |
2648 | QChar::Script script = items[i].script; |
2649 | if (script > QChar::Script_Khmer) |
2650 | script = QChar::Script_Common; |
2651 | CharAttributeFunction attributeFunction = charAttributeFunction[script]; |
2652 | if (!attributeFunction) |
2653 | continue; |
2654 | qsizetype end = i < numItems - 1 ? items[i + 1].position : stringLength; |
2655 | attributeFunction(script, string, items[i].position, end - items[i].position, attributes); |
2656 | } |
2657 | } |
2658 | |
2659 | } |
2660 | |
2661 | Q_CORE_EXPORT void initCharAttributes(QStringView string, |
2662 | const ScriptItem *items, qsizetype numItems, |
2663 | QCharAttributes *attributes, CharAttributeOptions options) |
2664 | { |
2665 | if (string.size() <= 0) |
2666 | return; |
2667 | |
2668 | if (!(options & DontClearAttributes)) |
2669 | ::memset(s: attributes, c: 0, n: (string.size() + 1) * sizeof(QCharAttributes)); |
2670 | |
2671 | if (options & GraphemeBreaks) |
2672 | getGraphemeBreaks(string: string.utf16(), len: string.size(), attributes); |
2673 | if (options & WordBreaks) |
2674 | getWordBreaks(string: string.utf16(), len: string.size(), attributes); |
2675 | if (options & SentenceBreaks) |
2676 | getSentenceBreaks(string: string.utf16(), len: string.size(), attributes); |
2677 | if (options & LineBreaks) |
2678 | getLineBreaks(string: string.utf16(), len: string.size(), attributes, options); |
2679 | if (options & WhiteSpaces) |
2680 | getWhiteSpaces(string: string.utf16(), len: string.size(), attributes); |
2681 | |
2682 | if (!qt_initcharattributes_default_algorithm_only) { |
2683 | if (!items || numItems <= 0) |
2684 | return; |
2685 | |
2686 | Tailored::getCharAttributes(string: string.utf16(), stringLength: string.size(), items, numItems, attributes); |
2687 | } |
2688 | } |
2689 | |
2690 | |
2691 | // ---------------------------------------------------------------------------- |
2692 | // |
2693 | // The Unicode script property. See http://www.unicode.org/reports/tr24/tr24-24.html |
2694 | // |
2695 | // ---------------------------------------------------------------------------- |
2696 | |
2697 | Q_CORE_EXPORT void initScripts(QStringView string, ScriptItemArray *scripts) |
2698 | { |
2699 | qsizetype sor = 0; |
2700 | qsizetype eor = 0; |
2701 | QChar::Script script = QChar::Script_Common; |
2702 | |
2703 | for (qsizetype i = 0; i < string.size(); ++i, eor = i) { |
2704 | char32_t ucs4 = string[i].unicode(); |
2705 | if (QChar::isHighSurrogate(ucs4) && i + 1 < string.size()) { |
2706 | ushort low = string[i + 1].unicode(); |
2707 | if (QChar::isLowSurrogate(ucs4: low)) { |
2708 | ucs4 = QChar::surrogateToUcs4(high: ucs4, low); |
2709 | ++i; |
2710 | } |
2711 | } |
2712 | |
2713 | const QUnicodeTables::Properties *prop = QUnicodeTables::properties(ucs4); |
2714 | |
2715 | QChar::Script nscript = QChar::Script(prop->script); |
2716 | |
2717 | if (Q_LIKELY(nscript == script || nscript <= QChar::Script_Common)) |
2718 | continue; |
2719 | |
2720 | // inherit preceding Common-s |
2721 | if (Q_UNLIKELY(script <= QChar::Script_Common)) { |
2722 | // also covers a case where the base character of Common script followed |
2723 | // by one or more combining marks of non-Inherited, non-Common script |
2724 | script = nscript; |
2725 | continue; |
2726 | } |
2727 | |
2728 | // Never break between a combining mark (gc= Mc, Mn or Me) and its base character. |
2729 | // Thus, a combining mark - whatever its script property value is - should inherit |
2730 | // the script property value of its base character. |
2731 | static const int test = (FLAG(QChar::Mark_NonSpacing) | FLAG(QChar::Mark_SpacingCombining) | FLAG(QChar::Mark_Enclosing)); |
2732 | if (Q_UNLIKELY(FLAG(prop->category) & test)) |
2733 | continue; |
2734 | |
2735 | Q_ASSERT(script > QChar::Script_Common); |
2736 | Q_ASSERT(sor < eor); |
2737 | scripts->append(t: ScriptItem{.position: sor, .script: script}); |
2738 | sor = eor; |
2739 | |
2740 | script = nscript; |
2741 | } |
2742 | |
2743 | Q_ASSERT(script >= QChar::Script_Common); |
2744 | Q_ASSERT(eor == string.size()); |
2745 | scripts->append(t: ScriptItem{.position: sor, .script: script}); |
2746 | } |
2747 | |
2748 | } // namespace QUnicodeTools |
2749 | |
2750 | QT_END_NAMESPACE |
2751 |
Definitions
- qt_initcharattributes_default_algorithm_only
- Extend_SpacingMark_ZWJ
- HardBreak
- breakTable
- shouldBreakBetweenClasses
- State
- getGraphemeBreaks
- Action
- breakTable
- getWordBreaks
- State
- breakTable
- getSentenceBreaks
- Action
- Class
- actionTable
- toClass
- DottedCircle
- State
- LinebreakUnit
- ParseState
- updateState
- Action
- breakTable
- getLineBreaks
- getWhiteSpaces
- Form
- indicForms
- form
- indic_nextSyllableBoundary
- indicAttributes
- thcell_t
- LibThai
- LibThai
- LibThai
- ~LibThai
- isInitialized
- brk_find_breaks
- next_cell
- g_libThai
- to_tis620
- thaiAssignAttributes
- thaiAttributes
- tibetanForm
- tibetan_nextSyllableBoundary
- tibetanAttributes
- MymrCharClassValues
- MymrCharClassFlags
- MymrChar
- mymrCharClasses
- getMyanmarCharClass
- mymrStateTable
- myanmar_nextSyllableBoundary
- myanmarAttributes
- KhmerCharClassValues
- KhmerCharClassFlags
- KhmerChar
- khmerCharClasses
- KhmerCharClassesRange
- getKhmerCharClass
- khmerStateTable
- khmer_nextSyllableBoundary
- khmerAttributes
- charAttributeFunction
- getCharAttributes
- initCharAttributes
Start learning QML with our Intro Training
Find out more