2
*******************************************************************************
4
* Copyright (C) 1999-2001, International Business Machines
5
* Corporation and others. All Rights Reserved.
7
*******************************************************************************
10
* tab size: 8 (not used)
13
* created on: 1999sep09
14
* created by: Markus W. Scherer
19
* \brief C API: UChar and UChar32 data types and UTF macros for C Unicode string handling
21
* <p>This file defines the UChar and UChar32 data types for Unicode code units
22
* and code points, as well as macros for efficiently getting code points
23
* in and out of a string.</p>
25
* <p>utf.h is included by utypes.h and itself includes the utfXX.h after some
26
* common definitions. Those files define the macros for each UTF-size.</p>
28
* <p>The original concept for these files was for ICU to allow
29
* in principle to set which UTF (UTF-8/16/32) is used internally
30
* by defining UTF_SIZE to either 8, 16, or 32. utf.h would then define the UChar type
31
* accordingly. UTF-16 was the default.</p>
33
* <p>This concept has been abandoned.
34
* A lot of the ICU source code — especially low-level code like
35
* conversion, normalization, and collation — assumes UTF-16,
36
* utf.h enforces the default of UTF-16.
37
* The UTF-8 and UTF-32 macros remain for now for completeness and backward compatibility.</p>
39
* <p>Accordingly, utf.h defines UChar to be an unsigned 16-bit integer. If this matches wchar_t, then
40
* UChar is defined to be exactly wchar_t, otherwise uint16_t.</p>
42
* <p>UChar32 is always defined to be a 32-bit integer to be large enough for a 21-bit
43
* Unicode code point (Unicode scalar value, 0..0x10ffff). If wchar_t is a 32-bit type, then
44
* UChar32 is defined to be exactly wchar_t, <em>regardless of whether wchar_t is signed or unsigned.
45
* This means that UChar32 may be signed or unsigned depending on the platform!</em>
46
* If wchar_t is not a 32-bit type, then UChar32 is defined to be uint32_t.</p>
48
* <p>utf.h also defines a number of C macros for handling single Unicode code points and
49
* for using UTF Unicode strings. It includes utf8.h, utf16.h, and utf32.h for the actual
50
* implementations of those macros and then aliases one set of them (for UTF-16) for general use.
51
* The UTF-specific macros have the UTF size in the macro name prefixes (UTF16_...), while
52
* the general alias macros always begin with UTF_...</p>
54
* <p>Many string operations can be done with or without error checking.
55
* Where such a distinction is useful, there are two versions of the macros, "unsafe" and "safe"
56
* ones with ..._UNSAFE and ..._SAFE suffixes. The unsafe macros are fast but may cause
57
* program failures if the strings are not well-formed. The safe macros have an additional, boolean
58
* parameter "strict". If strict is FALSE, then only illegal sequences are detected.
59
* Otherwise, irregular sequences and non-characters are detected as well (like single surrogates).
60
* Safe macros return special error code points for illegal/irregular sequences:
61
* Typically, U+ffff, or values that would result in a code unit sequence of the same length
62
* as the erroneous input sequence.<br>
63
* Note that _UNSAFE macros have fewer parameters: They do not have the strictness parameter, and
64
* they do not have start/length parameters for boundary checking.</p>
66
* <p>Here, the macros are aliased in two steps:
67
* In the first step, the UTF-specific macros with UTF16_ prefix and _UNSAFE and _SAFE suffixes are
68
* aliased according to the UTF_SIZE to macros with UTF_ prefix and the same suffixes and signatures.
69
* Then, in a second step, the default, general alias macros are set to use either the unsafe or
70
* the safe/not strict (default) or the safe/strict macro;
71
* these general macros do not have a strictness parameter.</p>
73
* <p>It is possible to change the default choice for the general alias macros to be unsafe, safe/not strict or safe/strict.
74
* The default is safe/not strict. It is not recommended to select the unsafe macros as the basis for
75
* Unicode string handling in ICU! To select this, define UTF_SAFE, UTF_STRICT, or UTF_UNSAFE.</p>
77
* <p>For general use, one should use the default, general macros with UTF_ prefix and no _SAFE/_UNSAFE suffix.
78
* Only in some cases it may be necessary to control the choice of macro directly and use a less generic alias.
79
* For example, if it can be assumed that a string is well-formed and the index will stay within the bounds,
80
* then the _UNSAFE version may be used.
81
* If a UTF-8 string is to be processed, then the macros with UTF8_ prefixes need to be used.</p>
82
* <p><b>Usage:</b> ICU coding guidelines for if() statements should be followed when using these macros.
83
* Compound statements (curly braces {}) must be used for if-else-while...
84
* bodies and all macro statements should be terminated with semicolon.</p>
92
* stddef.h defines wchar_t
95
#include "unicode/umachine.h"
96
/* include the utfXX.h after the following definitions */
98
/* If there is no compiler option for the preferred UTF size, then default to UTF-16. */
100
/** Number of bits in a Unicode string code unit, same as x in UTF-x (8, 16, or 32). */
104
/** Number of bytes in a UChar (sizeof(UChar)). */
105
#define U_SIZEOF_UCHAR (UTF_SIZE>>3)
108
* \def U_SIZEOF_WCHAR_T
109
* U_SIZEOF_WCHAR_T==sizeof(wchar_t).
111
#ifndef U_HAVE_WCHAR_H
112
# define U_HAVE_WCHAR_H 1
115
/* U_SIZEOF_WCHAR_T==sizeof(wchar_t) (0 means it is not defined or autoconf could not set it) */
116
#if U_SIZEOF_WCHAR_T==0
117
# undef U_SIZEOF_WCHAR_T
118
# define U_SIZEOF_WCHAR_T 4
122
* \def U_WCHAR_IS_UTF16
123
* Defined if wchar_t uses UTF-16.
126
* \def U_WCHAR_IS_UTF32
127
* Defined if wchar_t uses UTF-32.
129
#if !defined(U_WCHAR_IS_UTF16) && !defined(U_WCHAR_IS_UTF32)
130
# ifdef __STDC_ISO_10646__
131
# if (U_SIZEOF_WCHAR_T==2)
132
# define U_WCHAR_IS_UTF16
133
# elif (U_SIZEOF_WCHAR_T==4)
134
# define U_WCHAR_IS_UTF32
136
# elif defined __UCS2__
137
# if (__OS390__ || __OS400__) && (U_SIZEOF_WCHAR_T==2)
138
# define U_WCHAR_IS_UTF16
140
# elif defined __UCS4__
141
# if (U_SIZEOF_WCHAR_T==4)
142
# define U_WCHAR_IS_UTF32
144
# elif defined(WIN32) || defined(_WIN32) || defined(WIN64) || defined(_WIN64)
145
# define U_WCHAR_IS_UTF16
151
* Define UChar32 to be wchar_t if that is 32 bits wide; may be signed or unsigned!
152
* If wchar_t is not 32 bits wide, then define UChar32 to be uint32_t.
154
#if U_SIZEOF_WCHAR_T==4
155
typedef wchar_t UChar32;
157
typedef uint32_t UChar32;
161
* Unicode string and array offset and index type.
162
* ICU always counts Unicode code units (UChars) for string offsets, indexes, and lengths, not Unicode code points.
164
typedef int32_t UTextOffset;
166
/* Specify which macro versions are the default ones - safe or fast. */
167
#if !defined(UTF_SAFE) && !defined(UTF_STRICT) && !defined(UTF_UNSAFE)
169
* The default choice for general Unicode string macros is to use the ..._SAFE macro implementations
170
* with strict=FALSE. See the utf.h file description.
175
/* internal definitions ----------------------------------------------------- */
178
* <p>UTF8_ERROR_VALUE_1 and UTF8_ERROR_VALUE_2 are special error values for UTF-8,
179
* which need 1 or 2 bytes in UTF-8:<br>
180
* U+0015 = NAK = Negative Acknowledge, C0 control character<br>
181
* U+009f = highest C1 control character</p>
183
* <p>These are used by ("safe") UTF-8 macros so that they can return an error value
184
* that needs the same number of code units (bytes) as were seen by
185
* a macro. They should be tested with UTF_IS_ERROR() or UTF_IS_VALID().</p>
189
#define UTF8_ERROR_VALUE_1 0x15
190
#define UTF8_ERROR_VALUE_2 0x9f
193
* Error value for all UTFs. This code point value will be set by macros with error
194
* checking if an error is detected.
196
#define UTF_ERROR_VALUE 0xffff
198
/* single-code point definitions -------------------------------------------- */
200
/** Is this code unit or code point a surrogate (U+d800..U+dfff)? */
201
#define UTF_IS_SURROGATE(uchar) (((uchar)&0xfffff800)==0xd800)
204
* Is a given 32-bit code point a Unicode noncharacter?
206
#define UTF_IS_UNICODE_NONCHAR(c) \
207
(((((c) & 0xfffe) == 0xfffe) || ((c) >= 0xfdd0 && (c) <= 0xfdef)) && \
211
* Is a given 32-bit code point/Unicode scalar value
212
* actually a valid Unicode (abstract) character?
214
* Code points that are not characters include:
215
* - single surrogate code points (U+d800..U+dfff, 2048 code points)
216
* - the last two code points on each plane (U+__fffe and U+__ffff, 34 code points)
217
* - U+fdd0..U+fdef (new with Unicode 3.1, 32 code points)
218
* - the highest Unicode code point value is U+10ffff
220
* This means that all code points below U+d800 are character code points,
221
* and that boundary is tested first for performance.
223
#define UTF_IS_UNICODE_CHAR(c) \
224
((uint32_t)(c)<0xd800 || \
225
((uint32_t)(c)>0xdfff && \
226
(uint32_t)(c)<=0x10ffff && \
227
!UTF_IS_UNICODE_NONCHAR(c)))
230
* Is a given 32-bit code an error value
231
* as returned by one of the macros for any UTF?
233
#define UTF_IS_ERROR(c) \
234
(((c)&0xfffe)==0xfffe || (c)==UTF8_ERROR_VALUE_1 || (c)==UTF8_ERROR_VALUE_2)
236
/** This is a combined macro: Is c a valid Unicode value _and_ not an error code? */
237
#define UTF_IS_VALID(c) \
238
(UTF_IS_UNICODE_CHAR(c) && \
239
(c)!=UTF8_ERROR_VALUE_1 && (c)!=UTF8_ERROR_VALUE_2)
241
/* include the utfXX.h ------------------------------------------------------ */
243
#include "unicode/utf8.h"
244
#include "unicode/utf16.h"
245
#include "unicode/utf32.h"
247
/* Define types and macros according to the selected UTF size. -------------- */
251
* Define UChar to be wchar_t if that is 16 bits wide; always assumed to be unsigned.
252
* If wchar_t is not 16 bits wide, then define UChar to be uint16_t.
257
# error UTF-8 is not implemented, undefine UTF_SIZE or define it to 16
261
* limits.h defines CHAR_MAX
265
/* Define UChar to be compatible with char if possible. */
269
typedef uint8_t UChar;
274
/* Define UChar to be compatible with wchar_t if possible. */
275
# if U_SIZEOF_WCHAR_T==2
276
typedef wchar_t UChar;
278
typedef uint16_t UChar;
281
/** Does this code unit alone encode a code point? */
282
# define UTF_IS_SINGLE(uchar) UTF16_IS_SINGLE(uchar)
283
/** Is this code unit the first one of several? */
284
# define UTF_IS_LEAD(uchar) UTF16_IS_LEAD(uchar)
285
/** Is this code unit one of several but not the first one? */
286
# define UTF_IS_TRAIL(uchar) UTF16_IS_TRAIL(uchar)
288
/** Does this code point require multiple code units? */
289
# define UTF_NEED_MULTIPLE_UCHAR(c) UTF16_NEED_MULTIPLE_UCHAR(c)
290
/** How many code units are used to encode this code point? */
291
# define UTF_CHAR_LENGTH(c) UTF16_CHAR_LENGTH(c)
292
/** How many code units are used at most for any Unicode code point? */
293
# define UTF_MAX_CHAR_LENGTH UTF16_MAX_CHAR_LENGTH
294
/** Estimate the number of code units for a string based on the number of UTF-16 code units. */
295
# define UTF_ARRAY_SIZE(size) UTF16_ARRAY_SIZE(size)
297
/** See file documentation and UTF_GET_CHAR. */
298
# define UTF_GET_CHAR_UNSAFE(s, i, c) UTF16_GET_CHAR_UNSAFE(s, i, c)
299
/** See file documentation and UTF_GET_CHAR. */
300
# define UTF_GET_CHAR_SAFE(s, start, i, length, c, strict) UTF16_GET_CHAR_SAFE(s, start, i, length, c, strict)
302
/** See file documentation and UTF_NEXT_CHAR. */
303
# define UTF_NEXT_CHAR_UNSAFE(s, i, c) UTF16_NEXT_CHAR_UNSAFE(s, i, c)
304
/** See file documentation and UTF_NEXT_CHAR. */
305
# define UTF_NEXT_CHAR_SAFE(s, i, length, c, strict) UTF16_NEXT_CHAR_SAFE(s, i, length, c, strict)
307
/** See file documentation and UTF_APPEND_CHAR. */
308
# define UTF_APPEND_CHAR_UNSAFE(s, i, c) UTF16_APPEND_CHAR_UNSAFE(s, i, c)
309
/** See file documentation and UTF_APPEND_CHAR. */
310
# define UTF_APPEND_CHAR_SAFE(s, i, length, c) UTF16_APPEND_CHAR_SAFE(s, i, length, c)
312
/** See file documentation and UTF_FWD_1. */
313
# define UTF_FWD_1_UNSAFE(s, i) UTF16_FWD_1_UNSAFE(s, i)
314
/** See file documentation and UTF_FWD_1. */
315
# define UTF_FWD_1_SAFE(s, i, length) UTF16_FWD_1_SAFE(s, i, length)
317
/** See file documentation and UTF_FWD_N. */
318
# define UTF_FWD_N_UNSAFE(s, i, n) UTF16_FWD_N_UNSAFE(s, i, n)
319
/** See file documentation and UTF_FWD_N. */
320
# define UTF_FWD_N_SAFE(s, i, length, n) UTF16_FWD_N_SAFE(s, i, length, n)
322
/** See file documentation and UTF_SET_CHAR_START. */
323
# define UTF_SET_CHAR_START_UNSAFE(s, i) UTF16_SET_CHAR_START_UNSAFE(s, i)
324
/** See file documentation and UTF_SET_CHAR_START. */
325
# define UTF_SET_CHAR_START_SAFE(s, start, i) UTF16_SET_CHAR_START_SAFE(s, start, i)
327
/** See file documentation and UTF_PREV_CHAR. */
328
# define UTF_PREV_CHAR_UNSAFE(s, i, c) UTF16_PREV_CHAR_UNSAFE(s, i, c)
329
/** See file documentation and UTF_PREV_CHAR. */
330
# define UTF_PREV_CHAR_SAFE(s, start, i, c, strict) UTF16_PREV_CHAR_SAFE(s, start, i, c, strict)
332
/** See file documentation and UTF_BACK_1. */
333
# define UTF_BACK_1_UNSAFE(s, i) UTF16_BACK_1_UNSAFE(s, i)
334
/** See file documentation and UTF_BACK_1. */
335
# define UTF_BACK_1_SAFE(s, start, i) UTF16_BACK_1_SAFE(s, start, i)
337
/** See file documentation and UTF_BACK_N. */
338
# define UTF_BACK_N_UNSAFE(s, i, n) UTF16_BACK_N_UNSAFE(s, i, n)
339
/** See file documentation and UTF_BACK_N. */
340
# define UTF_BACK_N_SAFE(s, start, i, n) UTF16_BACK_N_SAFE(s, start, i, n)
342
/** See file documentation and UTF_SET_CHAR_LIMIT. */
343
# define UTF_SET_CHAR_LIMIT_UNSAFE(s, i) UTF16_SET_CHAR_LIMIT_UNSAFE(s, i)
344
/** See file documentation and UTF_SET_CHAR_LIMIT. */
345
# define UTF_SET_CHAR_LIMIT_SAFE(s, start, i, length) UTF16_SET_CHAR_LIMIT_SAFE(s, start, i, length)
349
# error UTF-32 is not implemented, undefine UTF_SIZE or define it to 16
351
typedef UChar32 UChar;
354
# error UTF_SIZE must be undefined or one of { 8, 16, 32 } - only 16 is implemented
357
/* Define the default macros for handling UTF characters. ------------------- */
360
* \def UTF_GET_CHAR(s, start, i, length, c)
362
* Set c to the code point that contains the code unit i.
363
* i could point to the first, the last, or an intermediate code unit.
369
* \def UTF_NEXT_CHAR(s, i, length, c)
371
* Set c to the code point that starts at code unit i
372
* and advance i to beyond the code units of this code point (post-increment).
373
* i must point to the first code unit of a code point.
379
* \def UTF_APPEND_CHAR(s, i, length, c)
381
* Append the code units of code point c to the string at index i
382
* and advance i to beyond the new code units (post-increment).
383
* The code units beginning at index i will be overwritten.
384
* \pre 0<=c<=0x10ffff
390
* \def UTF_FWD_1(s, i, length)
392
* Advance i to beyond the code units of the code point that begins at i.
393
* I.e., advance i by one code point.
394
* i must point to the first code unit of a code point.
400
* \def UTF_FWD_N(s, i, length, n)
402
* Advance i to beyond the code units of the n code points where the first one begins at i.
403
* I.e., advance i by n code points.
404
* i must point to the first code unit of a code point.
410
* \def UTF_SET_CHAR_START(s, start, i)
412
* Take the random-access index i and adjust it so that it points to the beginning
414
* The input index points to any code unit of a code point and is moved to point to
415
* the first code unit of the same code point. i is never incremented.
416
* This can be used to start an iteration with UTF_NEXT_CHAR() from a random index.
417
* \pre start<=i<length
418
* \post start<=i<length
422
* \def UTF_PREV_CHAR(s, start, i, c)
424
* Set c to the code point that has code units before i
425
* and move i backward (towards the beginning of the string)
426
* to the first code unit of this code point (pre-increment).
427
* i must point to the first code unit after the last unit of a code point (i==length is allowed).
428
* \pre start<i<=length
429
* \post start<=i<length
433
* \def UTF_BACK_1(s, start, i)
435
* Move i backward (towards the beginning of the string)
436
* to the first code unit of the code point that has code units before i.
437
* I.e., move i backward by one code point.
438
* i must point to the first code unit after the last unit of a code point (i==length is allowed).
439
* \pre start<i<=length
440
* \post start<=i<length
444
* \def UTF_BACK_N(s, start, i, n)
446
* Move i backward (towards the beginning of the string)
447
* to the first code unit of the n code points that have code units before i.
448
* I.e., move i backward by n code points.
449
* i must point to the first code unit after the last unit of a code point (i==length is allowed).
450
* \pre start<i<=length
451
* \post start<=i<length
455
* \def UTF_SET_CHAR_LIMIT(s, start, i, length)
457
* Take the random-access index i and adjust it so that it points beyond
458
* a code point. The input index points beyond any code unit
459
* of a code point and is moved to point beyond the last code unit of the same
460
* code point. i is never decremented.
461
* This can be used to start an iteration with UTF_PREV_CHAR() from a random index.
462
* \pre start<i<=length
463
* \post start<i<=length
468
# define UTF_GET_CHAR(s, start, i, length, c) UTF_GET_CHAR_SAFE(s, start, i, length, c, FALSE)
470
# define UTF_NEXT_CHAR(s, i, length, c) UTF_NEXT_CHAR_SAFE(s, i, length, c, FALSE)
471
# define UTF_APPEND_CHAR(s, i, length, c) UTF_APPEND_CHAR_SAFE(s, i, length, c)
472
# define UTF_FWD_1(s, i, length) UTF_FWD_1_SAFE(s, i, length)
473
# define UTF_FWD_N(s, i, length, n) UTF_FWD_N_SAFE(s, i, length, n)
474
# define UTF_SET_CHAR_START(s, start, i) UTF_SET_CHAR_START_SAFE(s, start, i)
476
# define UTF_PREV_CHAR(s, start, i, c) UTF_PREV_CHAR_SAFE(s, start, i, c, FALSE)
477
# define UTF_BACK_1(s, start, i) UTF_BACK_1_SAFE(s, start, i)
478
# define UTF_BACK_N(s, start, i, n) UTF_BACK_N_SAFE(s, start, i, n)
479
# define UTF_SET_CHAR_LIMIT(s, start, i, length) UTF_SET_CHAR_LIMIT_SAFE(s, start, i, length)
481
#elif defined(UTF_STRICT)
483
# define UTF_GET_CHAR(s, start, i, length, c) UTF_GET_CHAR_SAFE(s, start, i, length, c, TRUE)
485
# define UTF_NEXT_CHAR(s, i, length, c) UTF_NEXT_CHAR_SAFE(s, i, length, c, TRUE)
486
# define UTF_APPEND_CHAR(s, i, length, c) UTF_APPEND_CHAR_SAFE(s, i, length, c)
487
# define UTF_FWD_1(s, i, length) UTF_FWD_1_SAFE(s, i, length)
488
# define UTF_FWD_N(s, i, length, n) UTF_FWD_N_SAFE(s, i, length, n)
489
# define UTF_SET_CHAR_START(s, start, i) UTF_SET_CHAR_START_SAFE(s, start, i)
491
# define UTF_PREV_CHAR(s, start, i, c) UTF_PREV_CHAR_SAFE(s, start, i, c, TRUE)
492
# define UTF_BACK_1(s, start, i) UTF_BACK_1_SAFE(s, start, i)
493
# define UTF_BACK_N(s, start, i, n) UTF_BACK_N_SAFE(s, start, i, n)
494
# define UTF_SET_CHAR_LIMIT(s, start, i, length) UTF_SET_CHAR_LIMIT_SAFE(s, start, i, length)
496
#else /* UTF_UNSAFE */
498
# define UTF_GET_CHAR(s, start, i, length, c) UTF_GET_CHAR_UNSAFE(s, i, c)
500
# define UTF_NEXT_CHAR(s, i, length, c) UTF_NEXT_CHAR_UNSAFE(s, i, c)
501
# define UTF_APPEND_CHAR(s, i, length, c) UTF_APPEND_CHAR_UNSAFE(s, i, c)
502
# define UTF_FWD_1(s, i, length) UTF_FWD_1_UNSAFE(s, i)
503
# define UTF_FWD_N(s, i, length, n) UTF_FWD_N_UNSAFE(s, i, n)
504
# define UTF_SET_CHAR_START(s, start, i) UTF_SET_CHAR_START_UNSAFE(s, i)
506
# define UTF_PREV_CHAR(s, start, i, c) UTF_PREV_CHAR_UNSAFE(s, i, c)
507
# define UTF_BACK_1(s, start, i) UTF_BACK_1_UNSAFE(s, i)
508
# define UTF_BACK_N(s, start, i, n) UTF_BACK_N_UNSAFE(s, i, n)
509
# define UTF_SET_CHAR_LIMIT(s, start, i, length) UTF_SET_CHAR_LIMIT_UNSAFE(s, i)
added
removed
source/common/unicode/utf.h
