1// © 2016 and later: Unicode, Inc. and others.
2// License & terms of use: http://www.unicode.org/copyright.html
3/*
4*******************************************************************************
5* Copyright (c) 1996-2016, International Business Machines Corporation
6* and others. All Rights Reserved.
7*******************************************************************************
8* File unorm.h
9*
10* Created by: Vladimir Weinstein 12052000
11*
12* Modification history :
13*
14* Date Name Description
15* 02/01/01 synwee Added normalization quickcheck enum and method.
16*/
17#ifndef UNORM_H
18#define UNORM_H
19
20#include "unicode/utypes.h"
21
22#if !UCONFIG_NO_NORMALIZATION
23
24#include "unicode/uiter.h"
25#include "unicode/unorm2.h"
26
27/**
28 * \file
29 * \brief C API: Unicode Normalization
30 *
31 * Old Unicode normalization API.
32 *
33 * This API has been replaced by the unorm2.h API and is only available
34 * for backward compatibility. The functions here simply delegate to the
35 * unorm2.h functions, for example unorm2_getInstance() and unorm2_normalize().
36 * There is one exception: The new API does not provide a replacement for unorm_compare().
37 * Its declaration has been moved to unorm2.h.
38 *
39 * <code>unorm_normalize</code> transforms Unicode text into an equivalent composed or
40 * decomposed form, allowing for easier sorting and searching of text.
41 * <code>unorm_normalize</code> supports the standard normalization forms described in
42 * <a href="http://www.unicode.org/unicode/reports/tr15/" target="unicode">
43 * Unicode Standard Annex #15: Unicode Normalization Forms</a>.
44 *
45 * Characters with accents or other adornments can be encoded in
46 * several different ways in Unicode. For example, take the character A-acute.
47 * In Unicode, this can be encoded as a single character (the
48 * "composed" form):
49 *
50 * \code
51 * 00C1 LATIN CAPITAL LETTER A WITH ACUTE
52 * \endcode
53 *
54 * or as two separate characters (the "decomposed" form):
55 *
56 * \code
57 * 0041 LATIN CAPITAL LETTER A
58 * 0301 COMBINING ACUTE ACCENT
59 * \endcode
60 *
61 * To a user of your program, however, both of these sequences should be
62 * treated as the same "user-level" character "A with acute accent". When you are searching or
63 * comparing text, you must ensure that these two sequences are treated
64 * equivalently. In addition, you must handle characters with more than one
65 * accent. Sometimes the order of a character's combining accents is
66 * significant, while in other cases accent sequences in different orders are
67 * really equivalent.
68 *
69 * Similarly, the string "ffi" can be encoded as three separate letters:
70 *
71 * \code
72 * 0066 LATIN SMALL LETTER F
73 * 0066 LATIN SMALL LETTER F
74 * 0069 LATIN SMALL LETTER I
75 * \endcode
76 *
77 * or as the single character
78 *
79 * \code
80 * FB03 LATIN SMALL LIGATURE FFI
81 * \endcode
82 *
83 * The ffi ligature is not a distinct semantic character, and strictly speaking
84 * it shouldn't be in Unicode at all, but it was included for compatibility
85 * with existing character sets that already provided it. The Unicode standard
86 * identifies such characters by giving them "compatibility" decompositions
87 * into the corresponding semantic characters. When sorting and searching, you
88 * will often want to use these mappings.
89 *
90 * <code>unorm_normalize</code> helps solve these problems by transforming text into the
91 * canonical composed and decomposed forms as shown in the first example above.
92 * In addition, you can have it perform compatibility decompositions so that
93 * you can treat compatibility characters the same as their equivalents.
94 * Finally, <code>unorm_normalize</code> rearranges accents into the proper canonical
95 * order, so that you do not have to worry about accent rearrangement on your
96 * own.
97 *
98 * Form FCD, "Fast C or D", is also designed for collation.
99 * It allows to work on strings that are not necessarily normalized
100 * with an algorithm (like in collation) that works under "canonical closure", i.e., it treats precomposed
101 * characters and their decomposed equivalents the same.
102 *
103 * It is not a normalization form because it does not provide for uniqueness of representation. Multiple strings
104 * may be canonically equivalent (their NFDs are identical) and may all conform to FCD without being identical
105 * themselves.
106 *
107 * The form is defined such that the "raw decomposition", the recursive canonical decomposition of each character,
108 * results in a string that is canonically ordered. This means that precomposed characters are allowed for as long
109 * as their decompositions do not need canonical reordering.
110 *
111 * Its advantage for a process like collation is that all NFD and most NFC texts - and many unnormalized texts -
112 * already conform to FCD and do not need to be normalized (NFD) for such a process. The FCD quick check will
113 * return UNORM_YES for most strings in practice.
114 *
115 * unorm_normalize(UNORM_FCD) may be implemented with UNORM_NFD.
116 *
117 * For more details on FCD see the collation design document:
118 * https://htmlpreview.github.io/?https://github.com/unicode-org/icu-docs/blob/main/design/collation/ICU_collation_design.htm
119 *
120 * ICU collation performs either NFD or FCD normalization automatically if normalization
121 * is turned on for the collator object.
122 * Beyond collation and string search, normalized strings may be useful for string equivalence comparisons,
123 * transliteration/transcription, unique representations, etc.
124 *
125 * The W3C generally recommends to exchange texts in NFC.
126 * Note also that most legacy character encodings use only precomposed forms and often do not
127 * encode any combining marks by themselves. For conversion to such character encodings the
128 * Unicode text needs to be normalized to NFC.
129 * For more usage examples, see the Unicode Standard Annex.
130 */
131
132// Do not conditionalize the following enum with #ifndef U_HIDE_DEPRECATED_API,
133// it is needed for layout of Normalizer object.
134#ifndef U_FORCE_HIDE_DEPRECATED_API
135
136/**
137 * Constants for normalization modes.
138 * @deprecated ICU 56 Use unorm2.h instead.
139 */
140typedef enum {
141 /** No decomposition/composition. @deprecated ICU 56 Use unorm2.h instead. */
142 UNORM_NONE = 1,
143 /** Canonical decomposition. @deprecated ICU 56 Use unorm2.h instead. */
144 UNORM_NFD = 2,
145 /** Compatibility decomposition. @deprecated ICU 56 Use unorm2.h instead. */
146 UNORM_NFKD = 3,
147 /** Canonical decomposition followed by canonical composition. @deprecated ICU 56 Use unorm2.h instead. */
148 UNORM_NFC = 4,
149 /** Default normalization. @deprecated ICU 56 Use unorm2.h instead. */
150 UNORM_DEFAULT = UNORM_NFC,
151 /** Compatibility decomposition followed by canonical composition. @deprecated ICU 56 Use unorm2.h instead. */
152 UNORM_NFKC =5,
153 /** "Fast C or D" form. @deprecated ICU 56 Use unorm2.h instead. */
154 UNORM_FCD = 6,
155
156 /** One more than the highest normalization mode constant. @deprecated ICU 56 Use unorm2.h instead. */
157 UNORM_MODE_COUNT
158} UNormalizationMode;
159
160#endif // U_FORCE_HIDE_DEPRECATED_API
161
162#ifndef U_HIDE_DEPRECATED_API
163
164/**
165 * Constants for options flags for normalization.
166 * Use 0 for default options,
167 * including normalization according to the Unicode version
168 * that is currently supported by ICU (see u_getUnicodeVersion).
169 * @deprecated ICU 56 Use unorm2.h instead.
170 */
171enum {
172 /**
173 * Options bit set value to select Unicode 3.2 normalization
174 * (except NormalizationCorrections).
175 * At most one Unicode version can be selected at a time.
176 * @deprecated ICU 56 Use unorm2.h instead.
177 */
178 UNORM_UNICODE_3_2=0x20
179};
180
181/**
182 * Lowest-order bit number of unorm_compare() options bits corresponding to
183 * normalization options bits.
184 *
185 * The options parameter for unorm_compare() uses most bits for
186 * itself and for various comparison and folding flags.
187 * The most significant bits, however, are shifted down and passed on
188 * to the normalization implementation.
189 * (That is, from unorm_compare(..., options, ...),
190 * options>>UNORM_COMPARE_NORM_OPTIONS_SHIFT will be passed on to the
191 * internal normalization functions.)
192 *
193 * @see unorm_compare
194 * @deprecated ICU 56 Use unorm2.h instead.
195 */
196#define UNORM_COMPARE_NORM_OPTIONS_SHIFT 20
197
198/**
199 * Normalize a string.
200 * The string will be normalized according the specified normalization mode
201 * and options.
202 * The source and result buffers must not be the same, nor overlap.
203 *
204 * @param source The string to normalize.
205 * @param sourceLength The length of source, or -1 if NUL-terminated.
206 * @param mode The normalization mode; one of UNORM_NONE,
207 * UNORM_NFD, UNORM_NFC, UNORM_NFKC, UNORM_NFKD, UNORM_DEFAULT.
208 * @param options The normalization options, ORed together (0 for no options).
209 * @param result A pointer to a buffer to receive the result string.
210 * The result string is NUL-terminated if possible.
211 * @param resultLength The maximum size of result.
212 * @param status A pointer to a UErrorCode to receive any errors.
213 * @return The total buffer size needed; if greater than resultLength,
214 * the output was truncated, and the error code is set to U_BUFFER_OVERFLOW_ERROR.
215 * @deprecated ICU 56 Use unorm2.h instead.
216 */
217U_DEPRECATED int32_t U_EXPORT2
218unorm_normalize(const UChar *source, int32_t sourceLength,
219 UNormalizationMode mode, int32_t options,
220 UChar *result, int32_t resultLength,
221 UErrorCode *status);
222
223/**
224 * Performing quick check on a string, to quickly determine if the string is
225 * in a particular normalization format.
226 * Three types of result can be returned UNORM_YES, UNORM_NO or
227 * UNORM_MAYBE. Result UNORM_YES indicates that the argument
228 * string is in the desired normalized format, UNORM_NO determines that
229 * argument string is not in the desired normalized format. A
230 * UNORM_MAYBE result indicates that a more thorough check is required,
231 * the user may have to put the string in its normalized form and compare the
232 * results.
233 *
234 * @param source string for determining if it is in a normalized format
235 * @param sourcelength length of source to test, or -1 if NUL-terminated
236 * @param mode which normalization form to test for
237 * @param status a pointer to a UErrorCode to receive any errors
238 * @return UNORM_YES, UNORM_NO or UNORM_MAYBE
239 *
240 * @see unorm_isNormalized
241 * @deprecated ICU 56 Use unorm2.h instead.
242 */
243U_DEPRECATED UNormalizationCheckResult U_EXPORT2
244unorm_quickCheck(const UChar *source, int32_t sourcelength,
245 UNormalizationMode mode,
246 UErrorCode *status);
247
248/**
249 * Performing quick check on a string; same as unorm_quickCheck but
250 * takes an extra options parameter like most normalization functions.
251 *
252 * @param src String that is to be tested if it is in a normalization format.
253 * @param srcLength Length of source to test, or -1 if NUL-terminated.
254 * @param mode Which normalization form to test for.
255 * @param options The normalization options, ORed together (0 for no options).
256 * @param pErrorCode ICU error code in/out parameter.
257 * Must fulfill U_SUCCESS before the function call.
258 * @return UNORM_YES, UNORM_NO or UNORM_MAYBE
259 *
260 * @see unorm_quickCheck
261 * @see unorm_isNormalized
262 * @deprecated ICU 56 Use unorm2.h instead.
263 */
264U_DEPRECATED UNormalizationCheckResult U_EXPORT2
265unorm_quickCheckWithOptions(const UChar *src, int32_t srcLength,
266 UNormalizationMode mode, int32_t options,
267 UErrorCode *pErrorCode);
268
269/**
270 * Test if a string is in a given normalization form.
271 * This is semantically equivalent to source.equals(normalize(source, mode)) .
272 *
273 * Unlike unorm_quickCheck(), this function returns a definitive result,
274 * never a "maybe".
275 * For NFD, NFKD, and FCD, both functions work exactly the same.
276 * For NFC and NFKC where quickCheck may return "maybe", this function will
277 * perform further tests to arrive at a true/false result.
278 *
279 * @param src String that is to be tested if it is in a normalization format.
280 * @param srcLength Length of source to test, or -1 if NUL-terminated.
281 * @param mode Which normalization form to test for.
282 * @param pErrorCode ICU error code in/out parameter.
283 * Must fulfill U_SUCCESS before the function call.
284 * @return Boolean value indicating whether the source string is in the
285 * "mode" normalization form.
286 *
287 * @see unorm_quickCheck
288 * @deprecated ICU 56 Use unorm2.h instead.
289 */
290U_DEPRECATED UBool U_EXPORT2
291unorm_isNormalized(const UChar *src, int32_t srcLength,
292 UNormalizationMode mode,
293 UErrorCode *pErrorCode);
294
295/**
296 * Test if a string is in a given normalization form; same as unorm_isNormalized but
297 * takes an extra options parameter like most normalization functions.
298 *
299 * @param src String that is to be tested if it is in a normalization format.
300 * @param srcLength Length of source to test, or -1 if NUL-terminated.
301 * @param mode Which normalization form to test for.
302 * @param options The normalization options, ORed together (0 for no options).
303 * @param pErrorCode ICU error code in/out parameter.
304 * Must fulfill U_SUCCESS before the function call.
305 * @return Boolean value indicating whether the source string is in the
306 * "mode/options" normalization form.
307 *
308 * @see unorm_quickCheck
309 * @see unorm_isNormalized
310 * @deprecated ICU 56 Use unorm2.h instead.
311 */
312U_DEPRECATED UBool U_EXPORT2
313unorm_isNormalizedWithOptions(const UChar *src, int32_t srcLength,
314 UNormalizationMode mode, int32_t options,
315 UErrorCode *pErrorCode);
316
317/**
318 * Iterative normalization forward.
319 * This function (together with unorm_previous) is somewhat
320 * similar to the C++ Normalizer class (see its non-static functions).
321 *
322 * Iterative normalization is useful when only a small portion of a longer
323 * string/text needs to be processed.
324 *
325 * For example, the likelihood may be high that processing the first 10% of some
326 * text will be sufficient to find certain data.
327 * Another example: When one wants to concatenate two normalized strings and get a
328 * normalized result, it is much more efficient to normalize just a small part of
329 * the result around the concatenation place instead of re-normalizing everything.
330 *
331 * The input text is an instance of the C character iteration API UCharIterator.
332 * It may wrap around a simple string, a CharacterIterator, a Replaceable, or any
333 * other kind of text object.
334 *
335 * If a buffer overflow occurs, then the caller needs to reset the iterator to the
336 * old index and call the function again with a larger buffer - if the caller cares
337 * for the actual output.
338 * Regardless of the output buffer, the iterator will always be moved to the next
339 * normalization boundary.
340 *
341 * This function (like unorm_previous) serves two purposes:
342 *
343 * 1) To find the next boundary so that the normalization of the part of the text
344 * from the current position to that boundary does not affect and is not affected
345 * by the part of the text beyond that boundary.
346 *
347 * 2) To normalize the text up to the boundary.
348 *
349 * The second step is optional, per the doNormalize parameter.
350 * It is omitted for operations like string concatenation, where the two adjacent
351 * string ends need to be normalized together.
352 * In such a case, the output buffer will just contain a copy of the text up to the
353 * boundary.
354 *
355 * pNeededToNormalize is an output-only parameter. Its output value is only defined
356 * if normalization was requested (doNormalize) and successful (especially, no
357 * buffer overflow).
358 * It is useful for operations like a normalizing transliterator, where one would
359 * not want to replace a piece of text if it is not modified.
360 *
361 * If doNormalize==true and pNeededToNormalize!=NULL then *pNeeded... is set true
362 * if the normalization was necessary.
363 *
364 * If doNormalize==false then *pNeededToNormalize will be set to false.
365 *
366 * If the buffer overflows, then *pNeededToNormalize will be undefined;
367 * essentially, whenever U_FAILURE is true (like in buffer overflows), this result
368 * will be undefined.
369 *
370 * @param src The input text in the form of a C character iterator.
371 * @param dest The output buffer; can be NULL if destCapacity==0 for pure preflighting.
372 * @param destCapacity The number of UChars that fit into dest.
373 * @param mode The normalization mode.
374 * @param options The normalization options, ORed together (0 for no options).
375 * @param doNormalize Indicates if the source text up to the next boundary
376 * is to be normalized (true) or just copied (false).
377 * @param pNeededToNormalize Output flag indicating if the normalization resulted in
378 * different text from the input.
379 * Not defined if an error occurs including buffer overflow.
380 * Always false if !doNormalize.
381 * @param pErrorCode ICU error code in/out parameter.
382 * Must fulfill U_SUCCESS before the function call.
383 * @return Length of output (number of UChars) when successful or buffer overflow.
384 *
385 * @see unorm_previous
386 * @see unorm_normalize
387 *
388 * @deprecated ICU 56 Use unorm2.h instead.
389 */
390U_DEPRECATED int32_t U_EXPORT2
391unorm_next(UCharIterator *src,
392 UChar *dest, int32_t destCapacity,
393 UNormalizationMode mode, int32_t options,
394 UBool doNormalize, UBool *pNeededToNormalize,
395 UErrorCode *pErrorCode);
396
397/**
398 * Iterative normalization backward.
399 * This function (together with unorm_next) is somewhat
400 * similar to the C++ Normalizer class (see its non-static functions).
401 * For all details see unorm_next.
402 *
403 * @param src The input text in the form of a C character iterator.
404 * @param dest The output buffer; can be NULL if destCapacity==0 for pure preflighting.
405 * @param destCapacity The number of UChars that fit into dest.
406 * @param mode The normalization mode.
407 * @param options The normalization options, ORed together (0 for no options).
408 * @param doNormalize Indicates if the source text up to the next boundary
409 * is to be normalized (true) or just copied (false).
410 * @param pNeededToNormalize Output flag indicating if the normalization resulted in
411 * different text from the input.
412 * Not defined if an error occurs including buffer overflow.
413 * Always false if !doNormalize.
414 * @param pErrorCode ICU error code in/out parameter.
415 * Must fulfill U_SUCCESS before the function call.
416 * @return Length of output (number of UChars) when successful or buffer overflow.
417 *
418 * @see unorm_next
419 * @see unorm_normalize
420 *
421 * @deprecated ICU 56 Use unorm2.h instead.
422 */
423U_DEPRECATED int32_t U_EXPORT2
424unorm_previous(UCharIterator *src,
425 UChar *dest, int32_t destCapacity,
426 UNormalizationMode mode, int32_t options,
427 UBool doNormalize, UBool *pNeededToNormalize,
428 UErrorCode *pErrorCode);
429
430/**
431 * Concatenate normalized strings, making sure that the result is normalized as well.
432 *
433 * If both the left and the right strings are in
434 * the normalization form according to "mode/options",
435 * then the result will be
436 *
437 * \code
438 * dest=normalize(left+right, mode, options)
439 * \endcode
440 *
441 * With the input strings already being normalized,
442 * this function will use unorm_next() and unorm_previous()
443 * to find the adjacent end pieces of the input strings.
444 * Only the concatenation of these end pieces will be normalized and
445 * then concatenated with the remaining parts of the input strings.
446 *
447 * It is allowed to have dest==left to avoid copying the entire left string.
448 *
449 * @param left Left source string, may be same as dest.
450 * @param leftLength Length of left source string, or -1 if NUL-terminated.
451 * @param right Right source string. Must not be the same as dest, nor overlap.
452 * @param rightLength Length of right source string, or -1 if NUL-terminated.
453 * @param dest The output buffer; can be NULL if destCapacity==0 for pure preflighting.
454 * @param destCapacity The number of UChars that fit into dest.
455 * @param mode The normalization mode.
456 * @param options The normalization options, ORed together (0 for no options).
457 * @param pErrorCode ICU error code in/out parameter.
458 * Must fulfill U_SUCCESS before the function call.
459 * @return Length of output (number of UChars) when successful or buffer overflow.
460 *
461 * @see unorm_normalize
462 * @see unorm_next
463 * @see unorm_previous
464 *
465 * @deprecated ICU 56 Use unorm2.h instead.
466 */
467U_DEPRECATED int32_t U_EXPORT2
468unorm_concatenate(const UChar *left, int32_t leftLength,
469 const UChar *right, int32_t rightLength,
470 UChar *dest, int32_t destCapacity,
471 UNormalizationMode mode, int32_t options,
472 UErrorCode *pErrorCode);
473
474#endif /* U_HIDE_DEPRECATED_API */
475#endif /* #if !UCONFIG_NO_NORMALIZATION */
476#endif
477

source code of include/unicode/unorm.h