1
#ifndef Py_UNICODEOBJECT_H
2
#define Py_UNICODEOBJECT_H
8
Unicode implementation based on original code by Fredrik Lundh,
9
modified by Marc-Andre Lemburg (mal@lemburg.com) according to the
10
Unicode Integration Proposal (see file Misc/unicode.txt).
12
Copyright (c) Corporation for National Research Initiatives.
16
--------------------------------------------------------------------
18
* Yet another Unicode string type for Python. This type supports the
19
* 16-bit Basic Multilingual Plane (BMP) only.
21
* Written by Fredrik Lundh, January 1999.
23
* Copyright (c) 1999 by Secret Labs AB.
24
* Copyright (c) 1999 by Fredrik Lundh.
26
* fredrik@pythonware.com
27
* http://www.pythonware.com
29
* --------------------------------------------------------------------
30
* This Unicode String Type is
32
* Copyright (c) 1999 by Secret Labs AB
33
* Copyright (c) 1999 by Fredrik Lundh
35
* By obtaining, using, and/or copying this software and/or its
36
* associated documentation, you agree that you have read, understood,
37
* and will comply with the following terms and conditions:
39
* Permission to use, copy, modify, and distribute this software and its
40
* associated documentation for any purpose and without fee is hereby
41
* granted, provided that the above copyright notice appears in all
42
* copies, and that both that copyright notice and this permission notice
43
* appear in supporting documentation, and that the name of Secret Labs
44
* AB or the author not be used in advertising or publicity pertaining to
45
* distribution of the software without specific, written prior
48
* SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO
49
* THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
50
* FITNESS. IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR BE LIABLE FOR
51
* ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
52
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
53
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
54
* OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
55
* -------------------------------------------------------------------- */
59
/* === Internal API ======================================================= */
61
/* --- Internal Unicode Format -------------------------------------------- */
63
#ifndef Py_USING_UNICODE
65
#define PyUnicode_Check(op) 0
66
#define PyUnicode_CheckExact(op) 0
70
/* FIXME: MvL's new implementation assumes that Py_UNICODE_SIZE is
71
properly set, but the default rules below doesn't set it. I'll
72
sort this out some other day -- fredrik@pythonware.com */
74
#ifndef Py_UNICODE_SIZE
75
#error Must define Py_UNICODE_SIZE
78
/* Setting Py_UNICODE_WIDE enables UCS-4 storage. Otherwise, Unicode
79
strings are stored as UCS-2 (with limited support for UTF-16) */
81
#if Py_UNICODE_SIZE >= 4
82
#define Py_UNICODE_WIDE
85
/* Set these flags if the platform has "wchar.h", "wctype.h" and the
86
wchar_t type is a 16-bit unsigned type */
87
/* #define HAVE_WCHAR_H */
88
/* #define HAVE_USABLE_WCHAR_T */
90
/* Defaults for various platforms */
91
#ifndef PY_UNICODE_TYPE
93
/* Windows has a usable wchar_t type (unless we're using UCS-4) */
94
# if defined(MS_WIN32) && Py_UNICODE_SIZE == 2
95
# define HAVE_USABLE_WCHAR_T
96
# define PY_UNICODE_TYPE wchar_t
99
# if defined(Py_UNICODE_WIDE)
100
# define PY_UNICODE_TYPE Py_UCS4
105
/* If the compiler provides a wchar_t type we try to support it
106
through the interface functions PyUnicode_FromWideChar() and
107
PyUnicode_AsWideChar(). */
109
#ifdef HAVE_USABLE_WCHAR_T
110
# ifndef HAVE_WCHAR_H
111
# define HAVE_WCHAR_H
116
/* Work around a cosmetic bug in BSDI 4.x wchar.h; thanks to Thomas Wouters */
124
* Use this typedef when you need to represent a UTF-16 surrogate pair
125
* as single unsigned integer.
128
typedef unsigned int Py_UCS4;
129
#elif SIZEOF_LONG >= 4
130
typedef unsigned long Py_UCS4;
133
/* Py_UNICODE is the native Unicode storage format (code unit) used by
134
Python and represents a single Unicode element in the Unicode
137
typedef PY_UNICODE_TYPE Py_UNICODE;
139
/* --- UCS-2/UCS-4 Name Mangling ------------------------------------------ */
141
/* Unicode API names are mangled to assure that UCS-2 and UCS-4 builds
142
produce different external names and thus cause import errors in
143
case Python interpreters and extensions with mixed compiled in
144
Unicode width assumptions are combined. */
146
#ifndef Py_UNICODE_WIDE
148
# define PyUnicode_AsASCIIString PyUnicodeUCS2_AsASCIIString
149
# define PyUnicode_AsCharmapString PyUnicodeUCS2_AsCharmapString
150
# define PyUnicode_AsEncodedObject PyUnicodeUCS2_AsEncodedObject
151
# define PyUnicode_AsEncodedString PyUnicodeUCS2_AsEncodedString
152
# define PyUnicode_AsLatin1String PyUnicodeUCS2_AsLatin1String
153
# define PyUnicode_AsRawUnicodeEscapeString PyUnicodeUCS2_AsRawUnicodeEscapeString
154
# define PyUnicode_AsUTF32String PyUnicodeUCS2_AsUTF32String
155
# define PyUnicode_AsUTF16String PyUnicodeUCS2_AsUTF16String
156
# define PyUnicode_AsUTF8String PyUnicodeUCS2_AsUTF8String
157
# define PyUnicode_AsUnicode PyUnicodeUCS2_AsUnicode
158
# define PyUnicode_AsUnicodeEscapeString PyUnicodeUCS2_AsUnicodeEscapeString
159
# define PyUnicode_AsWideChar PyUnicodeUCS2_AsWideChar
160
# define PyUnicode_ClearFreeList PyUnicodeUCS2_ClearFreelist
161
# define PyUnicode_Compare PyUnicodeUCS2_Compare
162
# define PyUnicode_Concat PyUnicodeUCS2_Concat
163
# define PyUnicode_Contains PyUnicodeUCS2_Contains
164
# define PyUnicode_Count PyUnicodeUCS2_Count
165
# define PyUnicode_Decode PyUnicodeUCS2_Decode
166
# define PyUnicode_DecodeASCII PyUnicodeUCS2_DecodeASCII
167
# define PyUnicode_DecodeCharmap PyUnicodeUCS2_DecodeCharmap
168
# define PyUnicode_DecodeLatin1 PyUnicodeUCS2_DecodeLatin1
169
# define PyUnicode_DecodeRawUnicodeEscape PyUnicodeUCS2_DecodeRawUnicodeEscape
170
# define PyUnicode_DecodeUTF32 PyUnicodeUCS2_DecodeUTF32
171
# define PyUnicode_DecodeUTF32Stateful PyUnicodeUCS2_DecodeUTF32Stateful
172
# define PyUnicode_DecodeUTF16 PyUnicodeUCS2_DecodeUTF16
173
# define PyUnicode_DecodeUTF16Stateful PyUnicodeUCS2_DecodeUTF16Stateful
174
# define PyUnicode_DecodeUTF8 PyUnicodeUCS2_DecodeUTF8
175
# define PyUnicode_DecodeUTF8Stateful PyUnicodeUCS2_DecodeUTF8Stateful
176
# define PyUnicode_DecodeUnicodeEscape PyUnicodeUCS2_DecodeUnicodeEscape
177
# define PyUnicode_Encode PyUnicodeUCS2_Encode
178
# define PyUnicode_EncodeASCII PyUnicodeUCS2_EncodeASCII
179
# define PyUnicode_EncodeCharmap PyUnicodeUCS2_EncodeCharmap
180
# define PyUnicode_EncodeDecimal PyUnicodeUCS2_EncodeDecimal
181
# define PyUnicode_EncodeLatin1 PyUnicodeUCS2_EncodeLatin1
182
# define PyUnicode_EncodeRawUnicodeEscape PyUnicodeUCS2_EncodeRawUnicodeEscape
183
# define PyUnicode_EncodeUTF32 PyUnicodeUCS2_EncodeUTF32
184
# define PyUnicode_EncodeUTF16 PyUnicodeUCS2_EncodeUTF16
185
# define PyUnicode_EncodeUTF8 PyUnicodeUCS2_EncodeUTF8
186
# define PyUnicode_EncodeUnicodeEscape PyUnicodeUCS2_EncodeUnicodeEscape
187
# define PyUnicode_Find PyUnicodeUCS2_Find
188
# define PyUnicode_Format PyUnicodeUCS2_Format
189
# define PyUnicode_FromEncodedObject PyUnicodeUCS2_FromEncodedObject
190
# define PyUnicode_FromFormat PyUnicodeUCS2_FromFormat
191
# define PyUnicode_FromFormatV PyUnicodeUCS2_FromFormatV
192
# define PyUnicode_FromObject PyUnicodeUCS2_FromObject
193
# define PyUnicode_FromOrdinal PyUnicodeUCS2_FromOrdinal
194
# define PyUnicode_FromString PyUnicodeUCS2_FromString
195
# define PyUnicode_FromStringAndSize PyUnicodeUCS2_FromStringAndSize
196
# define PyUnicode_FromUnicode PyUnicodeUCS2_FromUnicode
197
# define PyUnicode_FromWideChar PyUnicodeUCS2_FromWideChar
198
# define PyUnicode_GetDefaultEncoding PyUnicodeUCS2_GetDefaultEncoding
199
# define PyUnicode_GetMax PyUnicodeUCS2_GetMax
200
# define PyUnicode_GetSize PyUnicodeUCS2_GetSize
201
# define PyUnicode_Join PyUnicodeUCS2_Join
202
# define PyUnicode_Partition PyUnicodeUCS2_Partition
203
# define PyUnicode_RPartition PyUnicodeUCS2_RPartition
204
# define PyUnicode_RSplit PyUnicodeUCS2_RSplit
205
# define PyUnicode_Replace PyUnicodeUCS2_Replace
206
# define PyUnicode_Resize PyUnicodeUCS2_Resize
207
# define PyUnicode_RichCompare PyUnicodeUCS2_RichCompare
208
# define PyUnicode_SetDefaultEncoding PyUnicodeUCS2_SetDefaultEncoding
209
# define PyUnicode_Split PyUnicodeUCS2_Split
210
# define PyUnicode_Splitlines PyUnicodeUCS2_Splitlines
211
# define PyUnicode_Tailmatch PyUnicodeUCS2_Tailmatch
212
# define PyUnicode_Translate PyUnicodeUCS2_Translate
213
# define PyUnicode_TranslateCharmap PyUnicodeUCS2_TranslateCharmap
214
# define _PyUnicode_AsDefaultEncodedString _PyUnicodeUCS2_AsDefaultEncodedString
215
# define _PyUnicode_Fini _PyUnicodeUCS2_Fini
216
# define _PyUnicode_Init _PyUnicodeUCS2_Init
217
# define _PyUnicode_IsAlpha _PyUnicodeUCS2_IsAlpha
218
# define _PyUnicode_IsDecimalDigit _PyUnicodeUCS2_IsDecimalDigit
219
# define _PyUnicode_IsDigit _PyUnicodeUCS2_IsDigit
220
# define _PyUnicode_IsLinebreak _PyUnicodeUCS2_IsLinebreak
221
# define _PyUnicode_IsLowercase _PyUnicodeUCS2_IsLowercase
222
# define _PyUnicode_IsNumeric _PyUnicodeUCS2_IsNumeric
223
# define _PyUnicode_IsTitlecase _PyUnicodeUCS2_IsTitlecase
224
# define _PyUnicode_IsUppercase _PyUnicodeUCS2_IsUppercase
225
# define _PyUnicode_IsWhitespace _PyUnicodeUCS2_IsWhitespace
226
# define _PyUnicode_ToDecimalDigit _PyUnicodeUCS2_ToDecimalDigit
227
# define _PyUnicode_ToDigit _PyUnicodeUCS2_ToDigit
228
# define _PyUnicode_ToLowercase _PyUnicodeUCS2_ToLowercase
229
# define _PyUnicode_ToNumeric _PyUnicodeUCS2_ToNumeric
230
# define _PyUnicode_ToTitlecase _PyUnicodeUCS2_ToTitlecase
231
# define _PyUnicode_ToUppercase _PyUnicodeUCS2_ToUppercase
235
# define PyUnicode_AsASCIIString PyUnicodeUCS4_AsASCIIString
236
# define PyUnicode_AsCharmapString PyUnicodeUCS4_AsCharmapString
237
# define PyUnicode_AsEncodedObject PyUnicodeUCS4_AsEncodedObject
238
# define PyUnicode_AsEncodedString PyUnicodeUCS4_AsEncodedString
239
# define PyUnicode_AsLatin1String PyUnicodeUCS4_AsLatin1String
240
# define PyUnicode_AsRawUnicodeEscapeString PyUnicodeUCS4_AsRawUnicodeEscapeString
241
# define PyUnicode_AsUTF32String PyUnicodeUCS4_AsUTF32String
242
# define PyUnicode_AsUTF16String PyUnicodeUCS4_AsUTF16String
243
# define PyUnicode_AsUTF8String PyUnicodeUCS4_AsUTF8String
244
# define PyUnicode_AsUnicode PyUnicodeUCS4_AsUnicode
245
# define PyUnicode_AsUnicodeEscapeString PyUnicodeUCS4_AsUnicodeEscapeString
246
# define PyUnicode_AsWideChar PyUnicodeUCS4_AsWideChar
247
# define PyUnicode_ClearFreeList PyUnicodeUCS4_ClearFreelist
248
# define PyUnicode_Compare PyUnicodeUCS4_Compare
249
# define PyUnicode_Concat PyUnicodeUCS4_Concat
250
# define PyUnicode_Contains PyUnicodeUCS4_Contains
251
# define PyUnicode_Count PyUnicodeUCS4_Count
252
# define PyUnicode_Decode PyUnicodeUCS4_Decode
253
# define PyUnicode_DecodeASCII PyUnicodeUCS4_DecodeASCII
254
# define PyUnicode_DecodeCharmap PyUnicodeUCS4_DecodeCharmap
255
# define PyUnicode_DecodeLatin1 PyUnicodeUCS4_DecodeLatin1
256
# define PyUnicode_DecodeRawUnicodeEscape PyUnicodeUCS4_DecodeRawUnicodeEscape
257
# define PyUnicode_DecodeUTF32 PyUnicodeUCS4_DecodeUTF32
258
# define PyUnicode_DecodeUTF32Stateful PyUnicodeUCS4_DecodeUTF32Stateful
259
# define PyUnicode_DecodeUTF16 PyUnicodeUCS4_DecodeUTF16
260
# define PyUnicode_DecodeUTF16Stateful PyUnicodeUCS4_DecodeUTF16Stateful
261
# define PyUnicode_DecodeUTF8 PyUnicodeUCS4_DecodeUTF8
262
# define PyUnicode_DecodeUTF8Stateful PyUnicodeUCS4_DecodeUTF8Stateful
263
# define PyUnicode_DecodeUnicodeEscape PyUnicodeUCS4_DecodeUnicodeEscape
264
# define PyUnicode_Encode PyUnicodeUCS4_Encode
265
# define PyUnicode_EncodeASCII PyUnicodeUCS4_EncodeASCII
266
# define PyUnicode_EncodeCharmap PyUnicodeUCS4_EncodeCharmap
267
# define PyUnicode_EncodeDecimal PyUnicodeUCS4_EncodeDecimal
268
# define PyUnicode_EncodeLatin1 PyUnicodeUCS4_EncodeLatin1
269
# define PyUnicode_EncodeRawUnicodeEscape PyUnicodeUCS4_EncodeRawUnicodeEscape
270
# define PyUnicode_EncodeUTF32 PyUnicodeUCS4_EncodeUTF32
271
# define PyUnicode_EncodeUTF16 PyUnicodeUCS4_EncodeUTF16
272
# define PyUnicode_EncodeUTF8 PyUnicodeUCS4_EncodeUTF8
273
# define PyUnicode_EncodeUnicodeEscape PyUnicodeUCS4_EncodeUnicodeEscape
274
# define PyUnicode_Find PyUnicodeUCS4_Find
275
# define PyUnicode_Format PyUnicodeUCS4_Format
276
# define PyUnicode_FromEncodedObject PyUnicodeUCS4_FromEncodedObject
277
# define PyUnicode_FromFormat PyUnicodeUCS4_FromFormat
278
# define PyUnicode_FromFormatV PyUnicodeUCS4_FromFormatV
279
# define PyUnicode_FromObject PyUnicodeUCS4_FromObject
280
# define PyUnicode_FromOrdinal PyUnicodeUCS4_FromOrdinal
281
# define PyUnicode_FromString PyUnicodeUCS4_FromString
282
# define PyUnicode_FromStringAndSize PyUnicodeUCS4_FromStringAndSize
283
# define PyUnicode_FromUnicode PyUnicodeUCS4_FromUnicode
284
# define PyUnicode_FromWideChar PyUnicodeUCS4_FromWideChar
285
# define PyUnicode_GetDefaultEncoding PyUnicodeUCS4_GetDefaultEncoding
286
# define PyUnicode_GetMax PyUnicodeUCS4_GetMax
287
# define PyUnicode_GetSize PyUnicodeUCS4_GetSize
288
# define PyUnicode_Join PyUnicodeUCS4_Join
289
# define PyUnicode_Partition PyUnicodeUCS4_Partition
290
# define PyUnicode_RPartition PyUnicodeUCS4_RPartition
291
# define PyUnicode_RSplit PyUnicodeUCS4_RSplit
292
# define PyUnicode_Replace PyUnicodeUCS4_Replace
293
# define PyUnicode_Resize PyUnicodeUCS4_Resize
294
# define PyUnicode_RichCompare PyUnicodeUCS4_RichCompare
295
# define PyUnicode_SetDefaultEncoding PyUnicodeUCS4_SetDefaultEncoding
296
# define PyUnicode_Split PyUnicodeUCS4_Split
297
# define PyUnicode_Splitlines PyUnicodeUCS4_Splitlines
298
# define PyUnicode_Tailmatch PyUnicodeUCS4_Tailmatch
299
# define PyUnicode_Translate PyUnicodeUCS4_Translate
300
# define PyUnicode_TranslateCharmap PyUnicodeUCS4_TranslateCharmap
301
# define _PyUnicode_AsDefaultEncodedString _PyUnicodeUCS4_AsDefaultEncodedString
302
# define _PyUnicode_Fini _PyUnicodeUCS4_Fini
303
# define _PyUnicode_Init _PyUnicodeUCS4_Init
304
# define _PyUnicode_IsAlpha _PyUnicodeUCS4_IsAlpha
305
# define _PyUnicode_IsDecimalDigit _PyUnicodeUCS4_IsDecimalDigit
306
# define _PyUnicode_IsDigit _PyUnicodeUCS4_IsDigit
307
# define _PyUnicode_IsLinebreak _PyUnicodeUCS4_IsLinebreak
308
# define _PyUnicode_IsLowercase _PyUnicodeUCS4_IsLowercase
309
# define _PyUnicode_IsNumeric _PyUnicodeUCS4_IsNumeric
310
# define _PyUnicode_IsTitlecase _PyUnicodeUCS4_IsTitlecase
311
# define _PyUnicode_IsUppercase _PyUnicodeUCS4_IsUppercase
312
# define _PyUnicode_IsWhitespace _PyUnicodeUCS4_IsWhitespace
313
# define _PyUnicode_ToDecimalDigit _PyUnicodeUCS4_ToDecimalDigit
314
# define _PyUnicode_ToDigit _PyUnicodeUCS4_ToDigit
315
# define _PyUnicode_ToLowercase _PyUnicodeUCS4_ToLowercase
316
# define _PyUnicode_ToNumeric _PyUnicodeUCS4_ToNumeric
317
# define _PyUnicode_ToTitlecase _PyUnicodeUCS4_ToTitlecase
318
# define _PyUnicode_ToUppercase _PyUnicodeUCS4_ToUppercase
323
/* --- Internal Unicode Operations ---------------------------------------- */
325
/* If you want Python to use the compiler's wctype.h functions instead
326
of the ones supplied with Python, define WANT_WCTYPE_FUNCTIONS or
327
configure Python using --with-wctype-functions. This reduces the
328
interpreter's code size. */
330
#if defined(HAVE_USABLE_WCHAR_T) && defined(WANT_WCTYPE_FUNCTIONS)
334
#define Py_UNICODE_ISSPACE(ch) iswspace(ch)
336
#define Py_UNICODE_ISLOWER(ch) iswlower(ch)
337
#define Py_UNICODE_ISUPPER(ch) iswupper(ch)
338
#define Py_UNICODE_ISTITLE(ch) _PyUnicode_IsTitlecase(ch)
339
#define Py_UNICODE_ISLINEBREAK(ch) _PyUnicode_IsLinebreak(ch)
341
#define Py_UNICODE_TOLOWER(ch) towlower(ch)
342
#define Py_UNICODE_TOUPPER(ch) towupper(ch)
343
#define Py_UNICODE_TOTITLE(ch) _PyUnicode_ToTitlecase(ch)
345
#define Py_UNICODE_ISDECIMAL(ch) _PyUnicode_IsDecimalDigit(ch)
346
#define Py_UNICODE_ISDIGIT(ch) _PyUnicode_IsDigit(ch)
347
#define Py_UNICODE_ISNUMERIC(ch) _PyUnicode_IsNumeric(ch)
349
#define Py_UNICODE_TODECIMAL(ch) _PyUnicode_ToDecimalDigit(ch)
350
#define Py_UNICODE_TODIGIT(ch) _PyUnicode_ToDigit(ch)
351
#define Py_UNICODE_TONUMERIC(ch) _PyUnicode_ToNumeric(ch)
353
#define Py_UNICODE_ISALPHA(ch) iswalpha(ch)
357
/* Since splitting on whitespace is an important use case, and
358
whitespace in most situations is solely ASCII whitespace, we
359
optimize for the common case by using a quick look-up table
360
_Py_ascii_whitespace (see below) with an inlined check.
363
#define Py_UNICODE_ISSPACE(ch) \
364
((ch) < 128U ? _Py_ascii_whitespace[(ch)] : _PyUnicode_IsWhitespace(ch))
366
#define Py_UNICODE_ISLOWER(ch) _PyUnicode_IsLowercase(ch)
367
#define Py_UNICODE_ISUPPER(ch) _PyUnicode_IsUppercase(ch)
368
#define Py_UNICODE_ISTITLE(ch) _PyUnicode_IsTitlecase(ch)
369
#define Py_UNICODE_ISLINEBREAK(ch) _PyUnicode_IsLinebreak(ch)
371
#define Py_UNICODE_TOLOWER(ch) _PyUnicode_ToLowercase(ch)
372
#define Py_UNICODE_TOUPPER(ch) _PyUnicode_ToUppercase(ch)
373
#define Py_UNICODE_TOTITLE(ch) _PyUnicode_ToTitlecase(ch)
375
#define Py_UNICODE_ISDECIMAL(ch) _PyUnicode_IsDecimalDigit(ch)
376
#define Py_UNICODE_ISDIGIT(ch) _PyUnicode_IsDigit(ch)
377
#define Py_UNICODE_ISNUMERIC(ch) _PyUnicode_IsNumeric(ch)
379
#define Py_UNICODE_TODECIMAL(ch) _PyUnicode_ToDecimalDigit(ch)
380
#define Py_UNICODE_TODIGIT(ch) _PyUnicode_ToDigit(ch)
381
#define Py_UNICODE_TONUMERIC(ch) _PyUnicode_ToNumeric(ch)
383
#define Py_UNICODE_ISALPHA(ch) _PyUnicode_IsAlpha(ch)
387
#define Py_UNICODE_ISALNUM(ch) \
388
(Py_UNICODE_ISALPHA(ch) || \
389
Py_UNICODE_ISDECIMAL(ch) || \
390
Py_UNICODE_ISDIGIT(ch) || \
391
Py_UNICODE_ISNUMERIC(ch))
393
#define Py_UNICODE_COPY(target, source, length) \
394
Py_MEMCPY((target), (source), (length)*sizeof(Py_UNICODE))
396
#define Py_UNICODE_FILL(target, value, length) \
397
do {Py_ssize_t i_; Py_UNICODE *t_ = (target); Py_UNICODE v_ = (value);\
398
for (i_ = 0; i_ < (length); i_++) t_[i_] = v_;\
401
/* Check if substring matches at given offset. the offset must be
402
valid, and the substring must not be empty */
404
#define Py_UNICODE_MATCH(string, offset, substring) \
405
((*((string)->str + (offset)) == *((substring)->str)) && \
406
((*((string)->str + (offset) + (substring)->length-1) == *((substring)->str + (substring)->length-1))) && \
407
!memcmp((string)->str + (offset), (substring)->str, (substring)->length*sizeof(Py_UNICODE)))
413
/* --- Unicode Type ------------------------------------------------------- */
417
Py_ssize_t length; /* Length of raw Unicode data in buffer */
418
Py_UNICODE *str; /* Raw Unicode buffer */
419
long hash; /* Hash value; -1 if not set */
420
PyObject *defenc; /* (Default) Encoded version as Python
421
string, or NULL; this is used for
422
implementing the buffer protocol */
425
PyAPI_DATA(PyTypeObject) PyUnicode_Type;
427
#define PyUnicode_Check(op) \
428
PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_UNICODE_SUBCLASS)
429
#define PyUnicode_CheckExact(op) (Py_TYPE(op) == &PyUnicode_Type)
431
/* Fast access macros */
432
#define PyUnicode_GET_SIZE(op) \
433
(((PyUnicodeObject *)(op))->length)
434
#define PyUnicode_GET_DATA_SIZE(op) \
435
(((PyUnicodeObject *)(op))->length * sizeof(Py_UNICODE))
436
#define PyUnicode_AS_UNICODE(op) \
437
(((PyUnicodeObject *)(op))->str)
438
#define PyUnicode_AS_DATA(op) \
439
((const char *)((PyUnicodeObject *)(op))->str)
441
/* --- Constants ---------------------------------------------------------- */
443
/* This Unicode character will be used as replacement character during
444
decoding if the errors argument is set to "replace". Note: the
445
Unicode character U+FFFD is the official REPLACEMENT CHARACTER in
448
#define Py_UNICODE_REPLACEMENT_CHARACTER ((Py_UNICODE) 0xFFFD)
450
/* === Public API ========================================================= */
452
/* --- Plain Py_UNICODE --------------------------------------------------- */
454
/* Create a Unicode Object from the Py_UNICODE buffer u of the given
457
u may be NULL which causes the contents to be undefined. It is the
458
user's responsibility to fill in the needed data afterwards. Note
459
that modifying the Unicode object contents after construction is
460
only allowed if u was set to NULL.
462
The buffer is copied into the new object. */
464
PyAPI_FUNC(PyObject*) PyUnicode_FromUnicode(
465
const Py_UNICODE *u, /* Unicode buffer */
466
Py_ssize_t size /* size of buffer */
469
/* Similar to PyUnicode_FromUnicode(), but u points to Latin-1 encoded bytes */
470
PyAPI_FUNC(PyObject*) PyUnicode_FromStringAndSize(
471
const char *u, /* char buffer */
472
Py_ssize_t size /* size of buffer */
475
/* Similar to PyUnicode_FromUnicode(), but u points to null-terminated
476
Latin-1 encoded bytes */
477
PyAPI_FUNC(PyObject*) PyUnicode_FromString(
478
const char *u /* string */
481
/* Return a read-only pointer to the Unicode object's internal
482
Py_UNICODE buffer. */
484
PyAPI_FUNC(Py_UNICODE *) PyUnicode_AsUnicode(
485
PyObject *unicode /* Unicode object */
488
/* Get the length of the Unicode object. */
490
PyAPI_FUNC(Py_ssize_t) PyUnicode_GetSize(
491
PyObject *unicode /* Unicode object */
494
/* Get the maximum ordinal for a Unicode character. */
495
PyAPI_FUNC(Py_UNICODE) PyUnicode_GetMax(void);
497
/* Resize an already allocated Unicode object to the new size length.
499
*unicode is modified to point to the new (resized) object and 0
502
This API may only be called by the function which also called the
503
Unicode constructor. The refcount on the object must be 1. Otherwise,
504
an error is returned.
506
Error handling is implemented as follows: an exception is set, -1
507
is returned and *unicode left untouched.
511
PyAPI_FUNC(int) PyUnicode_Resize(
512
PyObject **unicode, /* Pointer to the Unicode object */
513
Py_ssize_t length /* New length */
516
/* Coerce obj to an Unicode object and return a reference with
517
*incremented* refcount.
519
Coercion is done in the following way:
521
1. String and other char buffer compatible objects are decoded
522
under the assumptions that they contain data using the current
523
default encoding. Decoding is done in "strict" mode.
525
2. All other objects (including Unicode objects) raise an
528
The API returns NULL in case of an error. The caller is responsible
529
for decref'ing the returned objects.
533
PyAPI_FUNC(PyObject*) PyUnicode_FromEncodedObject(
534
register PyObject *obj, /* Object */
535
const char *encoding, /* encoding */
536
const char *errors /* error handling */
539
/* Coerce obj to an Unicode object and return a reference with
540
*incremented* refcount.
542
Unicode objects are passed back as-is (subclasses are converted to
543
true Unicode objects), all other objects are delegated to
544
PyUnicode_FromEncodedObject(obj, NULL, "strict") which results in
545
using the default encoding as basis for decoding the object.
547
The API returns NULL in case of an error. The caller is responsible
548
for decref'ing the returned objects.
552
PyAPI_FUNC(PyObject*) PyUnicode_FromObject(
553
register PyObject *obj /* Object */
556
PyAPI_FUNC(PyObject *) PyUnicode_FromFormatV(const char*, va_list);
557
PyAPI_FUNC(PyObject *) PyUnicode_FromFormat(const char*, ...);
559
/* Format the object based on the format_spec, as defined in PEP 3101
560
(Advanced String Formatting). */
561
PyAPI_FUNC(PyObject *) _PyUnicode_FormatAdvanced(PyObject *obj,
562
Py_UNICODE *format_spec,
563
Py_ssize_t format_spec_len);
565
/* --- wchar_t support for platforms which support it --------------------- */
569
/* Create a Unicode Object from the whcar_t buffer w of the given
572
The buffer is copied into the new object. */
574
PyAPI_FUNC(PyObject*) PyUnicode_FromWideChar(
575
register const wchar_t *w, /* wchar_t buffer */
576
Py_ssize_t size /* size of buffer */
579
/* Copies the Unicode Object contents into the wchar_t buffer w. At
580
most size wchar_t characters are copied.
582
Note that the resulting wchar_t string may or may not be
583
0-terminated. It is the responsibility of the caller to make sure
584
that the wchar_t string is 0-terminated in case this is required by
587
Returns the number of wchar_t characters copied (excluding a
588
possibly trailing 0-termination character) or -1 in case of an
591
PyAPI_FUNC(Py_ssize_t) PyUnicode_AsWideChar(
592
PyUnicodeObject *unicode, /* Unicode object */
593
register wchar_t *w, /* wchar_t buffer */
594
Py_ssize_t size /* size of buffer */
599
/* --- Unicode ordinals --------------------------------------------------- */
601
/* Create a Unicode Object from the given Unicode code point ordinal.
603
The ordinal must be in range(0x10000) on narrow Python builds
604
(UCS2), and range(0x110000) on wide builds (UCS4). A ValueError is
605
raised in case it is not.
609
PyAPI_FUNC(PyObject*) PyUnicode_FromOrdinal(int ordinal);
611
/* --- Free-list management ----------------------------------------------- */
613
/* Clear the free list used by the Unicode implementation.
615
This can be used to release memory used for objects on the free
616
list back to the Python memory allocator.
620
PyAPI_FUNC(int) PyUnicode_ClearFreeList(void);
622
/* === Builtin Codecs =====================================================
624
Many of these APIs take two arguments encoding and errors. These
625
parameters encoding and errors have the same semantics as the ones
626
of the builtin unicode() API.
628
Setting encoding to NULL causes the default encoding to be used.
630
Error handling is set by errors which may also be set to NULL
631
meaning to use the default handling defined for the codec. Default
632
error handling for all builtin codecs is "strict" (ValueErrors are
635
The codecs all use a similar interface. Only deviation from the
636
generic ones are documented.
640
/* --- Manage the default encoding ---------------------------------------- */
642
/* Return a Python string holding the default encoded value of the
645
The resulting string is cached in the Unicode object for subsequent
646
usage by this function. The cached version is needed to implement
647
the character buffer interface and will live (at least) as long as
648
the Unicode object itself.
650
The refcount of the string is *not* incremented.
652
*** Exported for internal use by the interpreter only !!! ***
656
PyAPI_FUNC(PyObject *) _PyUnicode_AsDefaultEncodedString(
657
PyObject *, const char *);
659
/* Returns the currently active default encoding.
661
The default encoding is currently implemented as run-time settable
662
process global. This may change in future versions of the
663
interpreter to become a parameter which is managed on a per-thread
668
PyAPI_FUNC(const char*) PyUnicode_GetDefaultEncoding(void);
670
/* Sets the currently active default encoding.
672
Returns 0 on success, -1 in case of an error.
676
PyAPI_FUNC(int) PyUnicode_SetDefaultEncoding(
677
const char *encoding /* Encoding name in standard form */
680
/* --- Generic Codecs ----------------------------------------------------- */
682
/* Create a Unicode object by decoding the encoded string s of the
685
PyAPI_FUNC(PyObject*) PyUnicode_Decode(
686
const char *s, /* encoded string */
687
Py_ssize_t size, /* size of buffer */
688
const char *encoding, /* encoding */
689
const char *errors /* error handling */
692
/* Encodes a Py_UNICODE buffer of the given size and returns a
693
Python string object. */
695
PyAPI_FUNC(PyObject*) PyUnicode_Encode(
696
const Py_UNICODE *s, /* Unicode char buffer */
697
Py_ssize_t size, /* number of Py_UNICODE chars to encode */
698
const char *encoding, /* encoding */
699
const char *errors /* error handling */
702
/* Encodes a Unicode object and returns the result as Python
705
PyAPI_FUNC(PyObject*) PyUnicode_AsEncodedObject(
706
PyObject *unicode, /* Unicode object */
707
const char *encoding, /* encoding */
708
const char *errors /* error handling */
711
/* Encodes a Unicode object and returns the result as Python string
714
PyAPI_FUNC(PyObject*) PyUnicode_AsEncodedString(
715
PyObject *unicode, /* Unicode object */
716
const char *encoding, /* encoding */
717
const char *errors /* error handling */
720
PyAPI_FUNC(PyObject*) PyUnicode_BuildEncodingMap(
721
PyObject* string /* 256 character map */
725
/* --- UTF-7 Codecs ------------------------------------------------------- */
727
PyAPI_FUNC(PyObject*) PyUnicode_DecodeUTF7(
728
const char *string, /* UTF-7 encoded string */
729
Py_ssize_t length, /* size of string */
730
const char *errors /* error handling */
733
PyAPI_FUNC(PyObject*) PyUnicode_DecodeUTF7Stateful(
734
const char *string, /* UTF-7 encoded string */
735
Py_ssize_t length, /* size of string */
736
const char *errors, /* error handling */
737
Py_ssize_t *consumed /* bytes consumed */
740
PyAPI_FUNC(PyObject*) PyUnicode_EncodeUTF7(
741
const Py_UNICODE *data, /* Unicode char buffer */
742
Py_ssize_t length, /* number of Py_UNICODE chars to encode */
743
int base64SetO, /* Encode RFC2152 Set O characters in base64 */
744
int base64WhiteSpace, /* Encode whitespace (sp, ht, nl, cr) in base64 */
745
const char *errors /* error handling */
748
/* --- UTF-8 Codecs ------------------------------------------------------- */
750
PyAPI_FUNC(PyObject*) PyUnicode_DecodeUTF8(
751
const char *string, /* UTF-8 encoded string */
752
Py_ssize_t length, /* size of string */
753
const char *errors /* error handling */
756
PyAPI_FUNC(PyObject*) PyUnicode_DecodeUTF8Stateful(
757
const char *string, /* UTF-8 encoded string */
758
Py_ssize_t length, /* size of string */
759
const char *errors, /* error handling */
760
Py_ssize_t *consumed /* bytes consumed */
763
PyAPI_FUNC(PyObject*) PyUnicode_AsUTF8String(
764
PyObject *unicode /* Unicode object */
767
PyAPI_FUNC(PyObject*) PyUnicode_EncodeUTF8(
768
const Py_UNICODE *data, /* Unicode char buffer */
769
Py_ssize_t length, /* number of Py_UNICODE chars to encode */
770
const char *errors /* error handling */
773
/* --- UTF-32 Codecs ------------------------------------------------------ */
775
/* Decodes length bytes from a UTF-32 encoded buffer string and returns
776
the corresponding Unicode object.
778
errors (if non-NULL) defines the error handling. It defaults
781
If byteorder is non-NULL, the decoder starts decoding using the
784
*byteorder == -1: little endian
785
*byteorder == 0: native order
786
*byteorder == 1: big endian
788
In native mode, the first four bytes of the stream are checked for a
789
BOM mark. If found, the BOM mark is analysed, the byte order
790
adjusted and the BOM skipped. In the other modes, no BOM mark
791
interpretation is done. After completion, *byteorder is set to the
792
current byte order at the end of input data.
794
If byteorder is NULL, the codec starts in native order mode.
798
PyAPI_FUNC(PyObject*) PyUnicode_DecodeUTF32(
799
const char *string, /* UTF-32 encoded string */
800
Py_ssize_t length, /* size of string */
801
const char *errors, /* error handling */
802
int *byteorder /* pointer to byteorder to use
803
0=native;-1=LE,1=BE; updated on
807
PyAPI_FUNC(PyObject*) PyUnicode_DecodeUTF32Stateful(
808
const char *string, /* UTF-32 encoded string */
809
Py_ssize_t length, /* size of string */
810
const char *errors, /* error handling */
811
int *byteorder, /* pointer to byteorder to use
812
0=native;-1=LE,1=BE; updated on
814
Py_ssize_t *consumed /* bytes consumed */
817
/* Returns a Python string using the UTF-32 encoding in native byte
818
order. The string always starts with a BOM mark. */
820
PyAPI_FUNC(PyObject*) PyUnicode_AsUTF32String(
821
PyObject *unicode /* Unicode object */
824
/* Returns a Python string object holding the UTF-32 encoded value of
827
If byteorder is not 0, output is written according to the following
830
byteorder == -1: little endian
831
byteorder == 0: native byte order (writes a BOM mark)
832
byteorder == 1: big endian
834
If byteorder is 0, the output string will always start with the
835
Unicode BOM mark (U+FEFF). In the other two modes, no BOM mark is
840
PyAPI_FUNC(PyObject*) PyUnicode_EncodeUTF32(
841
const Py_UNICODE *data, /* Unicode char buffer */
842
Py_ssize_t length, /* number of Py_UNICODE chars to encode */
843
const char *errors, /* error handling */
844
int byteorder /* byteorder to use 0=BOM+native;-1=LE,1=BE */
847
/* --- UTF-16 Codecs ------------------------------------------------------ */
849
/* Decodes length bytes from a UTF-16 encoded buffer string and returns
850
the corresponding Unicode object.
852
errors (if non-NULL) defines the error handling. It defaults
855
If byteorder is non-NULL, the decoder starts decoding using the
858
*byteorder == -1: little endian
859
*byteorder == 0: native order
860
*byteorder == 1: big endian
862
In native mode, the first two bytes of the stream are checked for a
863
BOM mark. If found, the BOM mark is analysed, the byte order
864
adjusted and the BOM skipped. In the other modes, no BOM mark
865
interpretation is done. After completion, *byteorder is set to the
866
current byte order at the end of input data.
868
If byteorder is NULL, the codec starts in native order mode.
872
PyAPI_FUNC(PyObject*) PyUnicode_DecodeUTF16(
873
const char *string, /* UTF-16 encoded string */
874
Py_ssize_t length, /* size of string */
875
const char *errors, /* error handling */
876
int *byteorder /* pointer to byteorder to use
877
0=native;-1=LE,1=BE; updated on
881
PyAPI_FUNC(PyObject*) PyUnicode_DecodeUTF16Stateful(
882
const char *string, /* UTF-16 encoded string */
883
Py_ssize_t length, /* size of string */
884
const char *errors, /* error handling */
885
int *byteorder, /* pointer to byteorder to use
886
0=native;-1=LE,1=BE; updated on
888
Py_ssize_t *consumed /* bytes consumed */
891
/* Returns a Python string using the UTF-16 encoding in native byte
892
order. The string always starts with a BOM mark. */
894
PyAPI_FUNC(PyObject*) PyUnicode_AsUTF16String(
895
PyObject *unicode /* Unicode object */
898
/* Returns a Python string object holding the UTF-16 encoded value of
901
If byteorder is not 0, output is written according to the following
904
byteorder == -1: little endian
905
byteorder == 0: native byte order (writes a BOM mark)
906
byteorder == 1: big endian
908
If byteorder is 0, the output string will always start with the
909
Unicode BOM mark (U+FEFF). In the other two modes, no BOM mark is
912
Note that Py_UNICODE data is being interpreted as UTF-16 reduced to
913
UCS-2. This trick makes it possible to add full UTF-16 capabilities
914
at a later point without compromising the APIs.
918
PyAPI_FUNC(PyObject*) PyUnicode_EncodeUTF16(
919
const Py_UNICODE *data, /* Unicode char buffer */
920
Py_ssize_t length, /* number of Py_UNICODE chars to encode */
921
const char *errors, /* error handling */
922
int byteorder /* byteorder to use 0=BOM+native;-1=LE,1=BE */
925
/* --- Unicode-Escape Codecs ---------------------------------------------- */
927
PyAPI_FUNC(PyObject*) PyUnicode_DecodeUnicodeEscape(
928
const char *string, /* Unicode-Escape encoded string */
929
Py_ssize_t length, /* size of string */
930
const char *errors /* error handling */
933
PyAPI_FUNC(PyObject*) PyUnicode_AsUnicodeEscapeString(
934
PyObject *unicode /* Unicode object */
937
PyAPI_FUNC(PyObject*) PyUnicode_EncodeUnicodeEscape(
938
const Py_UNICODE *data, /* Unicode char buffer */
939
Py_ssize_t length /* Number of Py_UNICODE chars to encode */
942
/* --- Raw-Unicode-Escape Codecs ------------------------------------------ */
944
PyAPI_FUNC(PyObject*) PyUnicode_DecodeRawUnicodeEscape(
945
const char *string, /* Raw-Unicode-Escape encoded string */
946
Py_ssize_t length, /* size of string */
947
const char *errors /* error handling */
950
PyAPI_FUNC(PyObject*) PyUnicode_AsRawUnicodeEscapeString(
951
PyObject *unicode /* Unicode object */
954
PyAPI_FUNC(PyObject*) PyUnicode_EncodeRawUnicodeEscape(
955
const Py_UNICODE *data, /* Unicode char buffer */
956
Py_ssize_t length /* Number of Py_UNICODE chars to encode */
959
/* --- Unicode Internal Codec ---------------------------------------------
961
Only for internal use in _codecsmodule.c */
963
PyObject *_PyUnicode_DecodeUnicodeInternal(
969
/* --- Latin-1 Codecs -----------------------------------------------------
971
Note: Latin-1 corresponds to the first 256 Unicode ordinals.
975
PyAPI_FUNC(PyObject*) PyUnicode_DecodeLatin1(
976
const char *string, /* Latin-1 encoded string */
977
Py_ssize_t length, /* size of string */
978
const char *errors /* error handling */
981
PyAPI_FUNC(PyObject*) PyUnicode_AsLatin1String(
982
PyObject *unicode /* Unicode object */
985
PyAPI_FUNC(PyObject*) PyUnicode_EncodeLatin1(
986
const Py_UNICODE *data, /* Unicode char buffer */
987
Py_ssize_t length, /* Number of Py_UNICODE chars to encode */
988
const char *errors /* error handling */
991
/* --- ASCII Codecs -------------------------------------------------------
993
Only 7-bit ASCII data is excepted. All other codes generate errors.
997
PyAPI_FUNC(PyObject*) PyUnicode_DecodeASCII(
998
const char *string, /* ASCII encoded string */
999
Py_ssize_t length, /* size of string */
1000
const char *errors /* error handling */
1003
PyAPI_FUNC(PyObject*) PyUnicode_AsASCIIString(
1004
PyObject *unicode /* Unicode object */
1007
PyAPI_FUNC(PyObject*) PyUnicode_EncodeASCII(
1008
const Py_UNICODE *data, /* Unicode char buffer */
1009
Py_ssize_t length, /* Number of Py_UNICODE chars to encode */
1010
const char *errors /* error handling */
1013
/* --- Character Map Codecs -----------------------------------------------
1015
This codec uses mappings to encode and decode characters.
1017
Decoding mappings must map single string characters to single
1018
Unicode characters, integers (which are then interpreted as Unicode
1019
ordinals) or None (meaning "undefined mapping" and causing an
1022
Encoding mappings must map single Unicode characters to single
1023
string characters, integers (which are then interpreted as Latin-1
1024
ordinals) or None (meaning "undefined mapping" and causing an
1027
If a character lookup fails with a LookupError, the character is
1028
copied as-is meaning that its ordinal value will be interpreted as
1029
Unicode or Latin-1 ordinal resp. Because of this mappings only need
1030
to contain those mappings which map characters to different code
1035
PyAPI_FUNC(PyObject*) PyUnicode_DecodeCharmap(
1036
const char *string, /* Encoded string */
1037
Py_ssize_t length, /* size of string */
1038
PyObject *mapping, /* character mapping
1039
(char ordinal -> unicode ordinal) */
1040
const char *errors /* error handling */
1043
PyAPI_FUNC(PyObject*) PyUnicode_AsCharmapString(
1044
PyObject *unicode, /* Unicode object */
1045
PyObject *mapping /* character mapping
1046
(unicode ordinal -> char ordinal) */
1049
PyAPI_FUNC(PyObject*) PyUnicode_EncodeCharmap(
1050
const Py_UNICODE *data, /* Unicode char buffer */
1051
Py_ssize_t length, /* Number of Py_UNICODE chars to encode */
1052
PyObject *mapping, /* character mapping
1053
(unicode ordinal -> char ordinal) */
1054
const char *errors /* error handling */
1057
/* Translate a Py_UNICODE buffer of the given length by applying a
1058
character mapping table to it and return the resulting Unicode
1061
The mapping table must map Unicode ordinal integers to Unicode
1062
ordinal integers or None (causing deletion of the character).
1064
Mapping tables may be dictionaries or sequences. Unmapped character
1065
ordinals (ones which cause a LookupError) are left untouched and
1070
PyAPI_FUNC(PyObject *) PyUnicode_TranslateCharmap(
1071
const Py_UNICODE *data, /* Unicode char buffer */
1072
Py_ssize_t length, /* Number of Py_UNICODE chars to encode */
1073
PyObject *table, /* Translate table */
1074
const char *errors /* error handling */
1079
/* --- MBCS codecs for Windows -------------------------------------------- */
1081
PyAPI_FUNC(PyObject*) PyUnicode_DecodeMBCS(
1082
const char *string, /* MBCS encoded string */
1083
Py_ssize_t length, /* size of string */
1084
const char *errors /* error handling */
1087
PyAPI_FUNC(PyObject*) PyUnicode_DecodeMBCSStateful(
1088
const char *string, /* MBCS encoded string */
1089
Py_ssize_t length, /* size of string */
1090
const char *errors, /* error handling */
1091
Py_ssize_t *consumed /* bytes consumed */
1094
PyAPI_FUNC(PyObject*) PyUnicode_AsMBCSString(
1095
PyObject *unicode /* Unicode object */
1098
PyAPI_FUNC(PyObject*) PyUnicode_EncodeMBCS(
1099
const Py_UNICODE *data, /* Unicode char buffer */
1100
Py_ssize_t length, /* Number of Py_UNICODE chars to encode */
1101
const char *errors /* error handling */
1104
#endif /* MS_WIN32 */
1106
/* --- Decimal Encoder ---------------------------------------------------- */
1108
/* Takes a Unicode string holding a decimal value and writes it into
1109
an output buffer using standard ASCII digit codes.
1111
The output buffer has to provide at least length+1 bytes of storage
1112
area. The output string is 0-terminated.
1114
The encoder converts whitespace to ' ', decimal characters to their
1115
corresponding ASCII digit and all other Latin-1 characters except
1116
\0 as-is. Characters outside this range (Unicode ordinals 1-256)
1117
are treated as errors. This includes embedded NULL bytes.
1119
Error handling is defined by the errors argument:
1121
NULL or "strict": raise a ValueError
1122
"ignore": ignore the wrong characters (these are not copied to the
1124
"replace": replaces illegal characters with '?'
1126
Returns 0 on success, -1 on failure.
1130
PyAPI_FUNC(int) PyUnicode_EncodeDecimal(
1131
Py_UNICODE *s, /* Unicode buffer */
1132
Py_ssize_t length, /* Number of Py_UNICODE chars to encode */
1133
char *output, /* Output buffer; must have size >= length */
1134
const char *errors /* error handling */
1137
/* --- Methods & Slots ----------------------------------------------------
1139
These are capable of handling Unicode objects and strings on input
1140
(we refer to them as strings in the descriptions) and return
1141
Unicode objects or integers as apporpriate. */
1143
/* Concat two strings giving a new Unicode string. */
1145
PyAPI_FUNC(PyObject*) PyUnicode_Concat(
1146
PyObject *left, /* Left string */
1147
PyObject *right /* Right string */
1150
/* Split a string giving a list of Unicode strings.
1152
If sep is NULL, splitting will be done at all whitespace
1153
substrings. Otherwise, splits occur at the given separator.
1155
At most maxsplit splits will be done. If negative, no limit is set.
1157
Separators are not included in the resulting list.
1161
PyAPI_FUNC(PyObject*) PyUnicode_Split(
1162
PyObject *s, /* String to split */
1163
PyObject *sep, /* String separator */
1164
Py_ssize_t maxsplit /* Maxsplit count */
1167
/* Dito, but split at line breaks.
1169
CRLF is considered to be one line break. Line breaks are not
1170
included in the resulting list. */
1172
PyAPI_FUNC(PyObject*) PyUnicode_Splitlines(
1173
PyObject *s, /* String to split */
1174
int keepends /* If true, line end markers are included */
1177
/* Partition a string using a given separator. */
1179
PyAPI_FUNC(PyObject*) PyUnicode_Partition(
1180
PyObject *s, /* String to partition */
1181
PyObject *sep /* String separator */
1184
/* Partition a string using a given separator, searching from the end of the
1187
PyAPI_FUNC(PyObject*) PyUnicode_RPartition(
1188
PyObject *s, /* String to partition */
1189
PyObject *sep /* String separator */
1192
/* Split a string giving a list of Unicode strings.
1194
If sep is NULL, splitting will be done at all whitespace
1195
substrings. Otherwise, splits occur at the given separator.
1197
At most maxsplit splits will be done. But unlike PyUnicode_Split
1198
PyUnicode_RSplit splits from the end of the string. If negative,
1201
Separators are not included in the resulting list.
1205
PyAPI_FUNC(PyObject*) PyUnicode_RSplit(
1206
PyObject *s, /* String to split */
1207
PyObject *sep, /* String separator */
1208
Py_ssize_t maxsplit /* Maxsplit count */
1211
/* Translate a string by applying a character mapping table to it and
1212
return the resulting Unicode object.
1214
The mapping table must map Unicode ordinal integers to Unicode
1215
ordinal integers or None (causing deletion of the character).
1217
Mapping tables may be dictionaries or sequences. Unmapped character
1218
ordinals (ones which cause a LookupError) are left untouched and
1223
PyAPI_FUNC(PyObject *) PyUnicode_Translate(
1224
PyObject *str, /* String */
1225
PyObject *table, /* Translate table */
1226
const char *errors /* error handling */
1229
/* Join a sequence of strings using the given separator and return
1230
the resulting Unicode string. */
1232
PyAPI_FUNC(PyObject*) PyUnicode_Join(
1233
PyObject *separator, /* Separator string */
1234
PyObject *seq /* Sequence object */
1237
/* Return 1 if substr matches str[start:end] at the given tail end, 0
1240
PyAPI_FUNC(Py_ssize_t) PyUnicode_Tailmatch(
1241
PyObject *str, /* String */
1242
PyObject *substr, /* Prefix or Suffix string */
1243
Py_ssize_t start, /* Start index */
1244
Py_ssize_t end, /* Stop index */
1245
int direction /* Tail end: -1 prefix, +1 suffix */
1248
/* Return the first position of substr in str[start:end] using the
1249
given search direction or -1 if not found. -2 is returned in case
1250
an error occurred and an exception is set. */
1252
PyAPI_FUNC(Py_ssize_t) PyUnicode_Find(
1253
PyObject *str, /* String */
1254
PyObject *substr, /* Substring to find */
1255
Py_ssize_t start, /* Start index */
1256
Py_ssize_t end, /* Stop index */
1257
int direction /* Find direction: +1 forward, -1 backward */
1260
/* Count the number of occurrences of substr in str[start:end]. */
1262
PyAPI_FUNC(Py_ssize_t) PyUnicode_Count(
1263
PyObject *str, /* String */
1264
PyObject *substr, /* Substring to count */
1265
Py_ssize_t start, /* Start index */
1266
Py_ssize_t end /* Stop index */
1269
/* Replace at most maxcount occurrences of substr in str with replstr
1270
and return the resulting Unicode object. */
1272
PyAPI_FUNC(PyObject *) PyUnicode_Replace(
1273
PyObject *str, /* String */
1274
PyObject *substr, /* Substring to find */
1275
PyObject *replstr, /* Substring to replace */
1276
Py_ssize_t maxcount /* Max. number of replacements to apply;
1280
/* Compare two strings and return -1, 0, 1 for less than, equal,
1281
greater than resp. */
1283
PyAPI_FUNC(int) PyUnicode_Compare(
1284
PyObject *left, /* Left string */
1285
PyObject *right /* Right string */
1288
/* Rich compare two strings and return one of the following:
1290
- NULL in case an exception was raised
1291
- Py_True or Py_False for successfuly comparisons
1292
- Py_NotImplemented in case the type combination is unknown
1294
Note that Py_EQ and Py_NE comparisons can cause a UnicodeWarning in
1295
case the conversion of the arguments to Unicode fails with a
1298
Possible values for op:
1300
Py_GT, Py_GE, Py_EQ, Py_NE, Py_LT, Py_LE
1304
PyAPI_FUNC(PyObject *) PyUnicode_RichCompare(
1305
PyObject *left, /* Left string */
1306
PyObject *right, /* Right string */
1307
int op /* Operation: Py_EQ, Py_NE, Py_GT, etc. */
1310
/* Apply an argument tuple or dictionary to a format string and return
1311
the resulting Unicode string. */
1313
PyAPI_FUNC(PyObject *) PyUnicode_Format(
1314
PyObject *format, /* Format string */
1315
PyObject *args /* Argument tuple or dictionary */
1318
/* Checks whether element is contained in container and return 1/0
1321
element has to coerce to an one element Unicode string. -1 is
1322
returned in case of an error. */
1324
PyAPI_FUNC(int) PyUnicode_Contains(
1325
PyObject *container, /* Container string */
1326
PyObject *element /* Element string */
1329
/* Externally visible for str.strip(unicode) */
1330
PyAPI_FUNC(PyObject *) _PyUnicode_XStrip(
1331
PyUnicodeObject *self,
1336
/* === Characters Type APIs =============================================== */
1338
/* Helper array used by Py_UNICODE_ISSPACE(). */
1340
PyAPI_DATA(const unsigned char) _Py_ascii_whitespace[];
1342
/* These should not be used directly. Use the Py_UNICODE_IS* and
1343
Py_UNICODE_TO* macros instead.
1345
These APIs are implemented in Objects/unicodectype.c.
1349
PyAPI_FUNC(int) _PyUnicode_IsLowercase(
1350
Py_UNICODE ch /* Unicode character */
1353
PyAPI_FUNC(int) _PyUnicode_IsUppercase(
1354
Py_UNICODE ch /* Unicode character */
1357
PyAPI_FUNC(int) _PyUnicode_IsTitlecase(
1358
Py_UNICODE ch /* Unicode character */
1361
PyAPI_FUNC(int) _PyUnicode_IsWhitespace(
1362
const Py_UNICODE ch /* Unicode character */
1365
PyAPI_FUNC(int) _PyUnicode_IsLinebreak(
1366
const Py_UNICODE ch /* Unicode character */
1369
PyAPI_FUNC(Py_UNICODE) _PyUnicode_ToLowercase(
1370
Py_UNICODE ch /* Unicode character */
1373
PyAPI_FUNC(Py_UNICODE) _PyUnicode_ToUppercase(
1374
Py_UNICODE ch /* Unicode character */
1377
PyAPI_FUNC(Py_UNICODE) _PyUnicode_ToTitlecase(
1378
Py_UNICODE ch /* Unicode character */
1381
PyAPI_FUNC(int) _PyUnicode_ToDecimalDigit(
1382
Py_UNICODE ch /* Unicode character */
1385
PyAPI_FUNC(int) _PyUnicode_ToDigit(
1386
Py_UNICODE ch /* Unicode character */
1389
PyAPI_FUNC(double) _PyUnicode_ToNumeric(
1390
Py_UNICODE ch /* Unicode character */
1393
PyAPI_FUNC(int) _PyUnicode_IsDecimalDigit(
1394
Py_UNICODE ch /* Unicode character */
1397
PyAPI_FUNC(int) _PyUnicode_IsDigit(
1398
Py_UNICODE ch /* Unicode character */
1401
PyAPI_FUNC(int) _PyUnicode_IsNumeric(
1402
Py_UNICODE ch /* Unicode character */
1405
PyAPI_FUNC(int) _PyUnicode_IsAlpha(
1406
Py_UNICODE ch /* Unicode character */
1412
#endif /* Py_USING_UNICODE */
1413
#endif /* !Py_UNICODEOBJECT_H */
1
#ifndef Py_UNICODEOBJECT_H
2
#define Py_UNICODEOBJECT_H
8
Unicode implementation based on original code by Fredrik Lundh,
9
modified by Marc-Andre Lemburg (mal@lemburg.com) according to the
10
Unicode Integration Proposal (see file Misc/unicode.txt).
12
Copyright (c) Corporation for National Research Initiatives.
16
--------------------------------------------------------------------
18
* Yet another Unicode string type for Python. This type supports the
19
* 16-bit Basic Multilingual Plane (BMP) only.
21
* Written by Fredrik Lundh, January 1999.
23
* Copyright (c) 1999 by Secret Labs AB.
24
* Copyright (c) 1999 by Fredrik Lundh.
26
* fredrik@pythonware.com
27
* http://www.pythonware.com
29
* --------------------------------------------------------------------
30
* This Unicode String Type is
32
* Copyright (c) 1999 by Secret Labs AB
33
* Copyright (c) 1999 by Fredrik Lundh
35
* By obtaining, using, and/or copying this software and/or its
36
* associated documentation, you agree that you have read, understood,
37
* and will comply with the following terms and conditions:
39
* Permission to use, copy, modify, and distribute this software and its
40
* associated documentation for any purpose and without fee is hereby
41
* granted, provided that the above copyright notice appears in all
42
* copies, and that both that copyright notice and this permission notice
43
* appear in supporting documentation, and that the name of Secret Labs
44
* AB or the author not be used in advertising or publicity pertaining to
45
* distribution of the software without specific, written prior
48
* SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO
49
* THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
50
* FITNESS. IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR BE LIABLE FOR
51
* ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
52
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
53
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
54
* OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
55
* -------------------------------------------------------------------- */
59
/* === Internal API ======================================================= */
61
/* --- Internal Unicode Format -------------------------------------------- */
63
#ifndef Py_USING_UNICODE
65
#define PyUnicode_Check(op) 0
66
#define PyUnicode_CheckExact(op) 0
70
/* FIXME: MvL's new implementation assumes that Py_UNICODE_SIZE is
71
properly set, but the default rules below doesn't set it. I'll
72
sort this out some other day -- fredrik@pythonware.com */
74
#ifndef Py_UNICODE_SIZE
75
#error Must define Py_UNICODE_SIZE
78
/* Setting Py_UNICODE_WIDE enables UCS-4 storage. Otherwise, Unicode
79
strings are stored as UCS-2 (with limited support for UTF-16) */
81
#if Py_UNICODE_SIZE >= 4
82
#define Py_UNICODE_WIDE
85
/* Set these flags if the platform has "wchar.h", "wctype.h" and the
86
wchar_t type is a 16-bit unsigned type */
87
/* #define HAVE_WCHAR_H */
88
/* #define HAVE_USABLE_WCHAR_T */
90
/* Defaults for various platforms */
91
#ifndef PY_UNICODE_TYPE
93
/* Windows has a usable wchar_t type (unless we're using UCS-4) */
94
# if defined(MS_WIN32) && Py_UNICODE_SIZE == 2
95
# define HAVE_USABLE_WCHAR_T
96
# define PY_UNICODE_TYPE wchar_t
99
# if defined(Py_UNICODE_WIDE)
100
# define PY_UNICODE_TYPE Py_UCS4
105
/* If the compiler provides a wchar_t type we try to support it
106
through the interface functions PyUnicode_FromWideChar() and
107
PyUnicode_AsWideChar(). */
109
#ifdef HAVE_USABLE_WCHAR_T
110
# ifndef HAVE_WCHAR_H
111
# define HAVE_WCHAR_H
116
/* Work around a cosmetic bug in BSDI 4.x wchar.h; thanks to Thomas Wouters */
124
* Use this typedef when you need to represent a UTF-16 surrogate pair
125
* as single unsigned integer.
128
typedef unsigned int Py_UCS4;
129
#elif SIZEOF_LONG >= 4
130
typedef unsigned long Py_UCS4;
133
/* Py_UNICODE is the native Unicode storage format (code unit) used by
134
Python and represents a single Unicode element in the Unicode
137
typedef PY_UNICODE_TYPE Py_UNICODE;
139
/* --- UCS-2/UCS-4 Name Mangling ------------------------------------------ */
141
/* Unicode API names are mangled to assure that UCS-2 and UCS-4 builds
142
produce different external names and thus cause import errors in
143
case Python interpreters and extensions with mixed compiled in
144
Unicode width assumptions are combined. */
146
#ifndef Py_UNICODE_WIDE
148
# define PyUnicode_AsASCIIString PyUnicodeUCS2_AsASCIIString
149
# define PyUnicode_AsCharmapString PyUnicodeUCS2_AsCharmapString
150
# define PyUnicode_AsEncodedObject PyUnicodeUCS2_AsEncodedObject
151
# define PyUnicode_AsEncodedString PyUnicodeUCS2_AsEncodedString
152
# define PyUnicode_AsLatin1String PyUnicodeUCS2_AsLatin1String
153
# define PyUnicode_AsRawUnicodeEscapeString PyUnicodeUCS2_AsRawUnicodeEscapeString
154
# define PyUnicode_AsUTF32String PyUnicodeUCS2_AsUTF32String
155
# define PyUnicode_AsUTF16String PyUnicodeUCS2_AsUTF16String
156
# define PyUnicode_AsUTF8String PyUnicodeUCS2_AsUTF8String
157
# define PyUnicode_AsUnicode PyUnicodeUCS2_AsUnicode
158
# define PyUnicode_AsUnicodeEscapeString PyUnicodeUCS2_AsUnicodeEscapeString
159
# define PyUnicode_AsWideChar PyUnicodeUCS2_AsWideChar
160
# define PyUnicode_ClearFreeList PyUnicodeUCS2_ClearFreelist
161
# define PyUnicode_Compare PyUnicodeUCS2_Compare
162
# define PyUnicode_Concat PyUnicodeUCS2_Concat
163
# define PyUnicode_Contains PyUnicodeUCS2_Contains
164
# define PyUnicode_Count PyUnicodeUCS2_Count
165
# define PyUnicode_Decode PyUnicodeUCS2_Decode
166
# define PyUnicode_DecodeASCII PyUnicodeUCS2_DecodeASCII
167
# define PyUnicode_DecodeCharmap PyUnicodeUCS2_DecodeCharmap
168
# define PyUnicode_DecodeLatin1 PyUnicodeUCS2_DecodeLatin1
169
# define PyUnicode_DecodeRawUnicodeEscape PyUnicodeUCS2_DecodeRawUnicodeEscape
170
# define PyUnicode_DecodeUTF32 PyUnicodeUCS2_DecodeUTF32
171
# define PyUnicode_DecodeUTF32Stateful PyUnicodeUCS2_DecodeUTF32Stateful
172
# define PyUnicode_DecodeUTF16 PyUnicodeUCS2_DecodeUTF16
173
# define PyUnicode_DecodeUTF16Stateful PyUnicodeUCS2_DecodeUTF16Stateful
174
# define PyUnicode_DecodeUTF8 PyUnicodeUCS2_DecodeUTF8
175
# define PyUnicode_DecodeUTF8Stateful PyUnicodeUCS2_DecodeUTF8Stateful
176
# define PyUnicode_DecodeUnicodeEscape PyUnicodeUCS2_DecodeUnicodeEscape
177
# define PyUnicode_Encode PyUnicodeUCS2_Encode
178
# define PyUnicode_EncodeASCII PyUnicodeUCS2_EncodeASCII
179
# define PyUnicode_EncodeCharmap PyUnicodeUCS2_EncodeCharmap
180
# define PyUnicode_EncodeDecimal PyUnicodeUCS2_EncodeDecimal
181
# define PyUnicode_EncodeLatin1 PyUnicodeUCS2_EncodeLatin1
182
# define PyUnicode_EncodeRawUnicodeEscape PyUnicodeUCS2_EncodeRawUnicodeEscape
183
# define PyUnicode_EncodeUTF32 PyUnicodeUCS2_EncodeUTF32
184
# define PyUnicode_EncodeUTF16 PyUnicodeUCS2_EncodeUTF16
185
# define PyUnicode_EncodeUTF8 PyUnicodeUCS2_EncodeUTF8
186
# define PyUnicode_EncodeUnicodeEscape PyUnicodeUCS2_EncodeUnicodeEscape
187
# define PyUnicode_Find PyUnicodeUCS2_Find
188
# define PyUnicode_Format PyUnicodeUCS2_Format
189
# define PyUnicode_FromEncodedObject PyUnicodeUCS2_FromEncodedObject
190
# define PyUnicode_FromFormat PyUnicodeUCS2_FromFormat
191
# define PyUnicode_FromFormatV PyUnicodeUCS2_FromFormatV
192
# define PyUnicode_FromObject PyUnicodeUCS2_FromObject
193
# define PyUnicode_FromOrdinal PyUnicodeUCS2_FromOrdinal
194
# define PyUnicode_FromString PyUnicodeUCS2_FromString
195
# define PyUnicode_FromStringAndSize PyUnicodeUCS2_FromStringAndSize
196
# define PyUnicode_FromUnicode PyUnicodeUCS2_FromUnicode
197
# define PyUnicode_FromWideChar PyUnicodeUCS2_FromWideChar
198
# define PyUnicode_GetDefaultEncoding PyUnicodeUCS2_GetDefaultEncoding
199
# define PyUnicode_GetMax PyUnicodeUCS2_GetMax
200
# define PyUnicode_GetSize PyUnicodeUCS2_GetSize
201
# define PyUnicode_Join PyUnicodeUCS2_Join
202
# define PyUnicode_Partition PyUnicodeUCS2_Partition
203
# define PyUnicode_RPartition PyUnicodeUCS2_RPartition
204
# define PyUnicode_RSplit PyUnicodeUCS2_RSplit
205
# define PyUnicode_Replace PyUnicodeUCS2_Replace
206
# define PyUnicode_Resize PyUnicodeUCS2_Resize
207
# define PyUnicode_RichCompare PyUnicodeUCS2_RichCompare
208
# define PyUnicode_SetDefaultEncoding PyUnicodeUCS2_SetDefaultEncoding
209
# define PyUnicode_Split PyUnicodeUCS2_Split
210
# define PyUnicode_Splitlines PyUnicodeUCS2_Splitlines
211
# define PyUnicode_Tailmatch PyUnicodeUCS2_Tailmatch
212
# define PyUnicode_Translate PyUnicodeUCS2_Translate
213
# define PyUnicode_TranslateCharmap PyUnicodeUCS2_TranslateCharmap
214
# define _PyUnicode_AsDefaultEncodedString _PyUnicodeUCS2_AsDefaultEncodedString
215
# define _PyUnicode_Fini _PyUnicodeUCS2_Fini
216
# define _PyUnicode_Init _PyUnicodeUCS2_Init
217
# define _PyUnicode_IsAlpha _PyUnicodeUCS2_IsAlpha
218
# define _PyUnicode_IsDecimalDigit _PyUnicodeUCS2_IsDecimalDigit
219
# define _PyUnicode_IsDigit _PyUnicodeUCS2_IsDigit
220
# define _PyUnicode_IsLinebreak _PyUnicodeUCS2_IsLinebreak
221
# define _PyUnicode_IsLowercase _PyUnicodeUCS2_IsLowercase
222
# define _PyUnicode_IsNumeric _PyUnicodeUCS2_IsNumeric
223
# define _PyUnicode_IsTitlecase _PyUnicodeUCS2_IsTitlecase
224
# define _PyUnicode_IsUppercase _PyUnicodeUCS2_IsUppercase
225
# define _PyUnicode_IsWhitespace _PyUnicodeUCS2_IsWhitespace
226
# define _PyUnicode_ToDecimalDigit _PyUnicodeUCS2_ToDecimalDigit
227
# define _PyUnicode_ToDigit _PyUnicodeUCS2_ToDigit
228
# define _PyUnicode_ToLowercase _PyUnicodeUCS2_ToLowercase
229
# define _PyUnicode_ToNumeric _PyUnicodeUCS2_ToNumeric
230
# define _PyUnicode_ToTitlecase _PyUnicodeUCS2_ToTitlecase
231
# define _PyUnicode_ToUppercase _PyUnicodeUCS2_ToUppercase
235
# define PyUnicode_AsASCIIString PyUnicodeUCS4_AsASCIIString
236
# define PyUnicode_AsCharmapString PyUnicodeUCS4_AsCharmapString
237
# define PyUnicode_AsEncodedObject PyUnicodeUCS4_AsEncodedObject
238
# define PyUnicode_AsEncodedString PyUnicodeUCS4_AsEncodedString
239
# define PyUnicode_AsLatin1String PyUnicodeUCS4_AsLatin1String
240
# define PyUnicode_AsRawUnicodeEscapeString PyUnicodeUCS4_AsRawUnicodeEscapeString
241
# define PyUnicode_AsUTF32String PyUnicodeUCS4_AsUTF32String
242
# define PyUnicode_AsUTF16String PyUnicodeUCS4_AsUTF16String
243
# define PyUnicode_AsUTF8String PyUnicodeUCS4_AsUTF8String
244
# define PyUnicode_AsUnicode PyUnicodeUCS4_AsUnicode
245
# define PyUnicode_AsUnicodeEscapeString PyUnicodeUCS4_AsUnicodeEscapeString
246
# define PyUnicode_AsWideChar PyUnicodeUCS4_AsWideChar
247
# define PyUnicode_ClearFreeList PyUnicodeUCS4_ClearFreelist
248
# define PyUnicode_Compare PyUnicodeUCS4_Compare
249
# define PyUnicode_Concat PyUnicodeUCS4_Concat
250
# define PyUnicode_Contains PyUnicodeUCS4_Contains
251
# define PyUnicode_Count PyUnicodeUCS4_Count
252
# define PyUnicode_Decode PyUnicodeUCS4_Decode
253
# define PyUnicode_DecodeASCII PyUnicodeUCS4_DecodeASCII
254
# define PyUnicode_DecodeCharmap PyUnicodeUCS4_DecodeCharmap
255
# define PyUnicode_DecodeLatin1 PyUnicodeUCS4_DecodeLatin1
256
# define PyUnicode_DecodeRawUnicodeEscape PyUnicodeUCS4_DecodeRawUnicodeEscape
257
# define PyUnicode_DecodeUTF32 PyUnicodeUCS4_DecodeUTF32
258
# define PyUnicode_DecodeUTF32Stateful PyUnicodeUCS4_DecodeUTF32Stateful
259
# define PyUnicode_DecodeUTF16 PyUnicodeUCS4_DecodeUTF16
260
# define PyUnicode_DecodeUTF16Stateful PyUnicodeUCS4_DecodeUTF16Stateful
261
# define PyUnicode_DecodeUTF8 PyUnicodeUCS4_DecodeUTF8
262
# define PyUnicode_DecodeUTF8Stateful PyUnicodeUCS4_DecodeUTF8Stateful
263
# define PyUnicode_DecodeUnicodeEscape PyUnicodeUCS4_DecodeUnicodeEscape
264
# define PyUnicode_Encode PyUnicodeUCS4_Encode
265
# define PyUnicode_EncodeASCII PyUnicodeUCS4_EncodeASCII
266
# define PyUnicode_EncodeCharmap PyUnicodeUCS4_EncodeCharmap
267
# define PyUnicode_EncodeDecimal PyUnicodeUCS4_EncodeDecimal
268
# define PyUnicode_EncodeLatin1 PyUnicodeUCS4_EncodeLatin1
269
# define PyUnicode_EncodeRawUnicodeEscape PyUnicodeUCS4_EncodeRawUnicodeEscape
270
# define PyUnicode_EncodeUTF32 PyUnicodeUCS4_EncodeUTF32
271
# define PyUnicode_EncodeUTF16 PyUnicodeUCS4_EncodeUTF16
272
# define PyUnicode_EncodeUTF8 PyUnicodeUCS4_EncodeUTF8
273
# define PyUnicode_EncodeUnicodeEscape PyUnicodeUCS4_EncodeUnicodeEscape
274
# define PyUnicode_Find PyUnicodeUCS4_Find
275
# define PyUnicode_Format PyUnicodeUCS4_Format
276
# define PyUnicode_FromEncodedObject PyUnicodeUCS4_FromEncodedObject
277
# define PyUnicode_FromFormat PyUnicodeUCS4_FromFormat
278
# define PyUnicode_FromFormatV PyUnicodeUCS4_FromFormatV
279
# define PyUnicode_FromObject PyUnicodeUCS4_FromObject
280
# define PyUnicode_FromOrdinal PyUnicodeUCS4_FromOrdinal
281
# define PyUnicode_FromString PyUnicodeUCS4_FromString
282
# define PyUnicode_FromStringAndSize PyUnicodeUCS4_FromStringAndSize
283
# define PyUnicode_FromUnicode PyUnicodeUCS4_FromUnicode
284
# define PyUnicode_FromWideChar PyUnicodeUCS4_FromWideChar
285
# define PyUnicode_GetDefaultEncoding PyUnicodeUCS4_GetDefaultEncoding
286
# define PyUnicode_GetMax PyUnicodeUCS4_GetMax
287
# define PyUnicode_GetSize PyUnicodeUCS4_GetSize
288
# define PyUnicode_Join PyUnicodeUCS4_Join
289
# define PyUnicode_Partition PyUnicodeUCS4_Partition
290
# define PyUnicode_RPartition PyUnicodeUCS4_RPartition
291
# define PyUnicode_RSplit PyUnicodeUCS4_RSplit
292
# define PyUnicode_Replace PyUnicodeUCS4_Replace
293
# define PyUnicode_Resize PyUnicodeUCS4_Resize
294
# define PyUnicode_RichCompare PyUnicodeUCS4_RichCompare
295
# define PyUnicode_SetDefaultEncoding PyUnicodeUCS4_SetDefaultEncoding
296
# define PyUnicode_Split PyUnicodeUCS4_Split
297
# define PyUnicode_Splitlines PyUnicodeUCS4_Splitlines
298
# define PyUnicode_Tailmatch PyUnicodeUCS4_Tailmatch
299
# define PyUnicode_Translate PyUnicodeUCS4_Translate
300
# define PyUnicode_TranslateCharmap PyUnicodeUCS4_TranslateCharmap
301
# define _PyUnicode_AsDefaultEncodedString _PyUnicodeUCS4_AsDefaultEncodedString
302
# define _PyUnicode_Fini _PyUnicodeUCS4_Fini
303
# define _PyUnicode_Init _PyUnicodeUCS4_Init
304
# define _PyUnicode_IsAlpha _PyUnicodeUCS4_IsAlpha
305
# define _PyUnicode_IsDecimalDigit _PyUnicodeUCS4_IsDecimalDigit
306
# define _PyUnicode_IsDigit _PyUnicodeUCS4_IsDigit
307
# define _PyUnicode_IsLinebreak _PyUnicodeUCS4_IsLinebreak
308
# define _PyUnicode_IsLowercase _PyUnicodeUCS4_IsLowercase
309
# define _PyUnicode_IsNumeric _PyUnicodeUCS4_IsNumeric
310
# define _PyUnicode_IsTitlecase _PyUnicodeUCS4_IsTitlecase
311
# define _PyUnicode_IsUppercase _PyUnicodeUCS4_IsUppercase
312
# define _PyUnicode_IsWhitespace _PyUnicodeUCS4_IsWhitespace
313
# define _PyUnicode_ToDecimalDigit _PyUnicodeUCS4_ToDecimalDigit
314
# define _PyUnicode_ToDigit _PyUnicodeUCS4_ToDigit
315
# define _PyUnicode_ToLowercase _PyUnicodeUCS4_ToLowercase
316
# define _PyUnicode_ToNumeric _PyUnicodeUCS4_ToNumeric
317
# define _PyUnicode_ToTitlecase _PyUnicodeUCS4_ToTitlecase
318
# define _PyUnicode_ToUppercase _PyUnicodeUCS4_ToUppercase
323
/* --- Internal Unicode Operations ---------------------------------------- */
325
/* If you want Python to use the compiler's wctype.h functions instead
326
of the ones supplied with Python, define WANT_WCTYPE_FUNCTIONS or
327
configure Python using --with-wctype-functions. This reduces the
328
interpreter's code size. */
330
#if defined(HAVE_USABLE_WCHAR_T) && defined(WANT_WCTYPE_FUNCTIONS)
334
#define Py_UNICODE_ISSPACE(ch) iswspace(ch)
336
#define Py_UNICODE_ISLOWER(ch) iswlower(ch)
337
#define Py_UNICODE_ISUPPER(ch) iswupper(ch)
338
#define Py_UNICODE_ISTITLE(ch) _PyUnicode_IsTitlecase(ch)
339
#define Py_UNICODE_ISLINEBREAK(ch) _PyUnicode_IsLinebreak(ch)
341
#define Py_UNICODE_TOLOWER(ch) towlower(ch)
342
#define Py_UNICODE_TOUPPER(ch) towupper(ch)
343
#define Py_UNICODE_TOTITLE(ch) _PyUnicode_ToTitlecase(ch)
345
#define Py_UNICODE_ISDECIMAL(ch) _PyUnicode_IsDecimalDigit(ch)
346
#define Py_UNICODE_ISDIGIT(ch) _PyUnicode_IsDigit(ch)
347
#define Py_UNICODE_ISNUMERIC(ch) _PyUnicode_IsNumeric(ch)
349
#define Py_UNICODE_TODECIMAL(ch) _PyUnicode_ToDecimalDigit(ch)
350
#define Py_UNICODE_TODIGIT(ch) _PyUnicode_ToDigit(ch)
351
#define Py_UNICODE_TONUMERIC(ch) _PyUnicode_ToNumeric(ch)
353
#define Py_UNICODE_ISALPHA(ch) iswalpha(ch)
357
/* Since splitting on whitespace is an important use case, and
358
whitespace in most situations is solely ASCII whitespace, we
359
optimize for the common case by using a quick look-up table
360
_Py_ascii_whitespace (see below) with an inlined check.
363
#define Py_UNICODE_ISSPACE(ch) \
364
((ch) < 128U ? _Py_ascii_whitespace[(ch)] : _PyUnicode_IsWhitespace(ch))
366
#define Py_UNICODE_ISLOWER(ch) _PyUnicode_IsLowercase(ch)
367
#define Py_UNICODE_ISUPPER(ch) _PyUnicode_IsUppercase(ch)
368
#define Py_UNICODE_ISTITLE(ch) _PyUnicode_IsTitlecase(ch)
369
#define Py_UNICODE_ISLINEBREAK(ch) _PyUnicode_IsLinebreak(ch)
371
#define Py_UNICODE_TOLOWER(ch) _PyUnicode_ToLowercase(ch)
372
#define Py_UNICODE_TOUPPER(ch) _PyUnicode_ToUppercase(ch)
373
#define Py_UNICODE_TOTITLE(ch) _PyUnicode_ToTitlecase(ch)
375
#define Py_UNICODE_ISDECIMAL(ch) _PyUnicode_IsDecimalDigit(ch)
376
#define Py_UNICODE_ISDIGIT(ch) _PyUnicode_IsDigit(ch)
377
#define Py_UNICODE_ISNUMERIC(ch) _PyUnicode_IsNumeric(ch)
379
#define Py_UNICODE_TODECIMAL(ch) _PyUnicode_ToDecimalDigit(ch)
380
#define Py_UNICODE_TODIGIT(ch) _PyUnicode_ToDigit(ch)
381
#define Py_UNICODE_TONUMERIC(ch) _PyUnicode_ToNumeric(ch)
383
#define Py_UNICODE_ISALPHA(ch) _PyUnicode_IsAlpha(ch)
387
#define Py_UNICODE_ISALNUM(ch) \
388
(Py_UNICODE_ISALPHA(ch) || \
389
Py_UNICODE_ISDECIMAL(ch) || \
390
Py_UNICODE_ISDIGIT(ch) || \
391
Py_UNICODE_ISNUMERIC(ch))
393
#define Py_UNICODE_COPY(target, source, length) \
394
Py_MEMCPY((target), (source), (length)*sizeof(Py_UNICODE))
396
#define Py_UNICODE_FILL(target, value, length) \
397
do {Py_ssize_t i_; Py_UNICODE *t_ = (target); Py_UNICODE v_ = (value);\
398
for (i_ = 0; i_ < (length); i_++) t_[i_] = v_;\
401
/* Check if substring matches at given offset. the offset must be
402
valid, and the substring must not be empty */
404
#define Py_UNICODE_MATCH(string, offset, substring) \
405
((*((string)->str + (offset)) == *((substring)->str)) && \
406
((*((string)->str + (offset) + (substring)->length-1) == *((substring)->str + (substring)->length-1))) && \
407
!memcmp((string)->str + (offset), (substring)->str, (substring)->length*sizeof(Py_UNICODE)))
413
/* --- Unicode Type ------------------------------------------------------- */
417
Py_ssize_t length; /* Length of raw Unicode data in buffer */
418
Py_UNICODE *str; /* Raw Unicode buffer */
419
long hash; /* Hash value; -1 if not set */
420
PyObject *defenc; /* (Default) Encoded version as Python
421
string, or NULL; this is used for
422
implementing the buffer protocol */
425
PyAPI_DATA(PyTypeObject) PyUnicode_Type;
427
#define PyUnicode_Check(op) \
428
PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_UNICODE_SUBCLASS)
429
#define PyUnicode_CheckExact(op) (Py_TYPE(op) == &PyUnicode_Type)
431
/* Fast access macros */
432
#define PyUnicode_GET_SIZE(op) \
433
(((PyUnicodeObject *)(op))->length)
434
#define PyUnicode_GET_DATA_SIZE(op) \
435
(((PyUnicodeObject *)(op))->length * sizeof(Py_UNICODE))
436
#define PyUnicode_AS_UNICODE(op) \
437
(((PyUnicodeObject *)(op))->str)
438
#define PyUnicode_AS_DATA(op) \
439
((const char *)((PyUnicodeObject *)(op))->str)
441
/* --- Constants ---------------------------------------------------------- */
443
/* This Unicode character will be used as replacement character during
444
decoding if the errors argument is set to "replace". Note: the
445
Unicode character U+FFFD is the official REPLACEMENT CHARACTER in
448
#define Py_UNICODE_REPLACEMENT_CHARACTER ((Py_UNICODE) 0xFFFD)
450
/* === Public API ========================================================= */
452
/* --- Plain Py_UNICODE --------------------------------------------------- */
454
/* Create a Unicode Object from the Py_UNICODE buffer u of the given
457
u may be NULL which causes the contents to be undefined. It is the
458
user's responsibility to fill in the needed data afterwards. Note
459
that modifying the Unicode object contents after construction is
460
only allowed if u was set to NULL.
462
The buffer is copied into the new object. */
464
PyAPI_FUNC(PyObject*) PyUnicode_FromUnicode(
465
const Py_UNICODE *u, /* Unicode buffer */
466
Py_ssize_t size /* size of buffer */
469
/* Similar to PyUnicode_FromUnicode(), but u points to Latin-1 encoded bytes */
470
PyAPI_FUNC(PyObject*) PyUnicode_FromStringAndSize(
471
const char *u, /* char buffer */
472
Py_ssize_t size /* size of buffer */
475
/* Similar to PyUnicode_FromUnicode(), but u points to null-terminated
476
Latin-1 encoded bytes */
477
PyAPI_FUNC(PyObject*) PyUnicode_FromString(
478
const char *u /* string */
481
/* Return a read-only pointer to the Unicode object's internal
482
Py_UNICODE buffer. */
484
PyAPI_FUNC(Py_UNICODE *) PyUnicode_AsUnicode(
485
PyObject *unicode /* Unicode object */
488
/* Get the length of the Unicode object. */
490
PyAPI_FUNC(Py_ssize_t) PyUnicode_GetSize(
491
PyObject *unicode /* Unicode object */
494
/* Get the maximum ordinal for a Unicode character. */
495
PyAPI_FUNC(Py_UNICODE) PyUnicode_GetMax(void);
497
/* Resize an already allocated Unicode object to the new size length.
499
*unicode is modified to point to the new (resized) object and 0
502
This API may only be called by the function which also called the
503
Unicode constructor. The refcount on the object must be 1. Otherwise,
504
an error is returned.
506
Error handling is implemented as follows: an exception is set, -1
507
is returned and *unicode left untouched.
511
PyAPI_FUNC(int) PyUnicode_Resize(
512
PyObject **unicode, /* Pointer to the Unicode object */
513
Py_ssize_t length /* New length */
516
/* Coerce obj to a Unicode object and return a reference with
517
*incremented* refcount.
519
Coercion is done in the following way:
521
1. String and other char buffer compatible objects are decoded
522
under the assumptions that they contain data using the current
523
default encoding. Decoding is done in "strict" mode.
525
2. All other objects (including Unicode objects) raise an
528
The API returns NULL in case of an error. The caller is responsible
529
for decref'ing the returned objects.
533
PyAPI_FUNC(PyObject*) PyUnicode_FromEncodedObject(
534
register PyObject *obj, /* Object */
535
const char *encoding, /* encoding */
536
const char *errors /* error handling */
539
/* Coerce obj to a Unicode object and return a reference with
540
*incremented* refcount.
542
Unicode objects are passed back as-is (subclasses are converted to
543
true Unicode objects), all other objects are delegated to
544
PyUnicode_FromEncodedObject(obj, NULL, "strict") which results in
545
using the default encoding as basis for decoding the object.
547
The API returns NULL in case of an error. The caller is responsible
548
for decref'ing the returned objects.
552
PyAPI_FUNC(PyObject*) PyUnicode_FromObject(
553
register PyObject *obj /* Object */
556
PyAPI_FUNC(PyObject *) PyUnicode_FromFormatV(const char*, va_list);
557
PyAPI_FUNC(PyObject *) PyUnicode_FromFormat(const char*, ...);
559
/* Format the object based on the format_spec, as defined in PEP 3101
560
(Advanced String Formatting). */
561
PyAPI_FUNC(PyObject *) _PyUnicode_FormatAdvanced(PyObject *obj,
562
Py_UNICODE *format_spec,
563
Py_ssize_t format_spec_len);
565
/* --- wchar_t support for platforms which support it --------------------- */
569
/* Create a Unicode Object from the whcar_t buffer w of the given
572
The buffer is copied into the new object. */
574
PyAPI_FUNC(PyObject*) PyUnicode_FromWideChar(
575
register const wchar_t *w, /* wchar_t buffer */
576
Py_ssize_t size /* size of buffer */
579
/* Copies the Unicode Object contents into the wchar_t buffer w. At
580
most size wchar_t characters are copied.
582
Note that the resulting wchar_t string may or may not be
583
0-terminated. It is the responsibility of the caller to make sure
584
that the wchar_t string is 0-terminated in case this is required by
587
Returns the number of wchar_t characters copied (excluding a
588
possibly trailing 0-termination character) or -1 in case of an
591
PyAPI_FUNC(Py_ssize_t) PyUnicode_AsWideChar(
592
PyUnicodeObject *unicode, /* Unicode object */
593
register wchar_t *w, /* wchar_t buffer */
594
Py_ssize_t size /* size of buffer */
599
/* --- Unicode ordinals --------------------------------------------------- */
601
/* Create a Unicode Object from the given Unicode code point ordinal.
603
The ordinal must be in range(0x10000) on narrow Python builds
604
(UCS2), and range(0x110000) on wide builds (UCS4). A ValueError is
605
raised in case it is not.
609
PyAPI_FUNC(PyObject*) PyUnicode_FromOrdinal(int ordinal);
611
/* --- Free-list management ----------------------------------------------- */
613
/* Clear the free list used by the Unicode implementation.
615
This can be used to release memory used for objects on the free
616
list back to the Python memory allocator.
620
PyAPI_FUNC(int) PyUnicode_ClearFreeList(void);
622
/* === Builtin Codecs =====================================================
624
Many of these APIs take two arguments encoding and errors. These
625
parameters encoding and errors have the same semantics as the ones
626
of the builtin unicode() API.
628
Setting encoding to NULL causes the default encoding to be used.
630
Error handling is set by errors which may also be set to NULL
631
meaning to use the default handling defined for the codec. Default
632
error handling for all builtin codecs is "strict" (ValueErrors are
635
The codecs all use a similar interface. Only deviation from the
636
generic ones are documented.
640
/* --- Manage the default encoding ---------------------------------------- */
642
/* Return a Python string holding the default encoded value of the
645
The resulting string is cached in the Unicode object for subsequent
646
usage by this function. The cached version is needed to implement
647
the character buffer interface and will live (at least) as long as
648
the Unicode object itself.
650
The refcount of the string is *not* incremented.
652
*** Exported for internal use by the interpreter only !!! ***
656
PyAPI_FUNC(PyObject *) _PyUnicode_AsDefaultEncodedString(
657
PyObject *, const char *);
659
/* Returns the currently active default encoding.
661
The default encoding is currently implemented as run-time settable
662
process global. This may change in future versions of the
663
interpreter to become a parameter which is managed on a per-thread
668
PyAPI_FUNC(const char*) PyUnicode_GetDefaultEncoding(void);
670
/* Sets the currently active default encoding.
672
Returns 0 on success, -1 in case of an error.
676
PyAPI_FUNC(int) PyUnicode_SetDefaultEncoding(
677
const char *encoding /* Encoding name in standard form */
680
/* --- Generic Codecs ----------------------------------------------------- */
682
/* Create a Unicode object by decoding the encoded string s of the
685
PyAPI_FUNC(PyObject*) PyUnicode_Decode(
686
const char *s, /* encoded string */
687
Py_ssize_t size, /* size of buffer */
688
const char *encoding, /* encoding */
689
const char *errors /* error handling */
692
/* Encodes a Py_UNICODE buffer of the given size and returns a
693
Python string object. */
695
PyAPI_FUNC(PyObject*) PyUnicode_Encode(
696
const Py_UNICODE *s, /* Unicode char buffer */
697
Py_ssize_t size, /* number of Py_UNICODE chars to encode */
698
const char *encoding, /* encoding */
699
const char *errors /* error handling */
702
/* Encodes a Unicode object and returns the result as Python
705
PyAPI_FUNC(PyObject*) PyUnicode_AsEncodedObject(
706
PyObject *unicode, /* Unicode object */
707
const char *encoding, /* encoding */
708
const char *errors /* error handling */
711
/* Encodes a Unicode object and returns the result as Python string
714
PyAPI_FUNC(PyObject*) PyUnicode_AsEncodedString(
715
PyObject *unicode, /* Unicode object */
716
const char *encoding, /* encoding */
717
const char *errors /* error handling */
720
PyAPI_FUNC(PyObject*) PyUnicode_BuildEncodingMap(
721
PyObject* string /* 256 character map */
725
/* --- UTF-7 Codecs ------------------------------------------------------- */
727
PyAPI_FUNC(PyObject*) PyUnicode_DecodeUTF7(
728
const char *string, /* UTF-7 encoded string */
729
Py_ssize_t length, /* size of string */
730
const char *errors /* error handling */
733
PyAPI_FUNC(PyObject*) PyUnicode_DecodeUTF7Stateful(
734
const char *string, /* UTF-7 encoded string */
735
Py_ssize_t length, /* size of string */
736
const char *errors, /* error handling */
737
Py_ssize_t *consumed /* bytes consumed */
740
PyAPI_FUNC(PyObject*) PyUnicode_EncodeUTF7(
741
const Py_UNICODE *data, /* Unicode char buffer */
742
Py_ssize_t length, /* number of Py_UNICODE chars to encode */
743
int base64SetO, /* Encode RFC2152 Set O characters in base64 */
744
int base64WhiteSpace, /* Encode whitespace (sp, ht, nl, cr) in base64 */
745
const char *errors /* error handling */
748
/* --- UTF-8 Codecs ------------------------------------------------------- */
750
PyAPI_FUNC(PyObject*) PyUnicode_DecodeUTF8(
751
const char *string, /* UTF-8 encoded string */
752
Py_ssize_t length, /* size of string */
753
const char *errors /* error handling */
756
PyAPI_FUNC(PyObject*) PyUnicode_DecodeUTF8Stateful(
757
const char *string, /* UTF-8 encoded string */
758
Py_ssize_t length, /* size of string */
759
const char *errors, /* error handling */
760
Py_ssize_t *consumed /* bytes consumed */
763
PyAPI_FUNC(PyObject*) PyUnicode_AsUTF8String(
764
PyObject *unicode /* Unicode object */
767
PyAPI_FUNC(PyObject*) PyUnicode_EncodeUTF8(
768
const Py_UNICODE *data, /* Unicode char buffer */
769
Py_ssize_t length, /* number of Py_UNICODE chars to encode */
770
const char *errors /* error handling */
773
/* --- UTF-32 Codecs ------------------------------------------------------ */
775
/* Decodes length bytes from a UTF-32 encoded buffer string and returns
776
the corresponding Unicode object.
778
errors (if non-NULL) defines the error handling. It defaults
781
If byteorder is non-NULL, the decoder starts decoding using the
784
*byteorder == -1: little endian
785
*byteorder == 0: native order
786
*byteorder == 1: big endian
788
In native mode, the first four bytes of the stream are checked for a
789
BOM mark. If found, the BOM mark is analysed, the byte order
790
adjusted and the BOM skipped. In the other modes, no BOM mark
791
interpretation is done. After completion, *byteorder is set to the
792
current byte order at the end of input data.
794
If byteorder is NULL, the codec starts in native order mode.
798
PyAPI_FUNC(PyObject*) PyUnicode_DecodeUTF32(
799
const char *string, /* UTF-32 encoded string */
800
Py_ssize_t length, /* size of string */
801
const char *errors, /* error handling */
802
int *byteorder /* pointer to byteorder to use
803
0=native;-1=LE,1=BE; updated on
807
PyAPI_FUNC(PyObject*) PyUnicode_DecodeUTF32Stateful(
808
const char *string, /* UTF-32 encoded string */
809
Py_ssize_t length, /* size of string */
810
const char *errors, /* error handling */
811
int *byteorder, /* pointer to byteorder to use
812
0=native;-1=LE,1=BE; updated on
814
Py_ssize_t *consumed /* bytes consumed */
817
/* Returns a Python string using the UTF-32 encoding in native byte
818
order. The string always starts with a BOM mark. */
820
PyAPI_FUNC(PyObject*) PyUnicode_AsUTF32String(
821
PyObject *unicode /* Unicode object */
824
/* Returns a Python string object holding the UTF-32 encoded value of
827
If byteorder is not 0, output is written according to the following
830
byteorder == -1: little endian
831
byteorder == 0: native byte order (writes a BOM mark)
832
byteorder == 1: big endian
834
If byteorder is 0, the output string will always start with the
835
Unicode BOM mark (U+FEFF). In the other two modes, no BOM mark is
840
PyAPI_FUNC(PyObject*) PyUnicode_EncodeUTF32(
841
const Py_UNICODE *data, /* Unicode char buffer */
842
Py_ssize_t length, /* number of Py_UNICODE chars to encode */
843
const char *errors, /* error handling */
844
int byteorder /* byteorder to use 0=BOM+native;-1=LE,1=BE */
847
/* --- UTF-16 Codecs ------------------------------------------------------ */
849
/* Decodes length bytes from a UTF-16 encoded buffer string and returns
850
the corresponding Unicode object.
852
errors (if non-NULL) defines the error handling. It defaults
855
If byteorder is non-NULL, the decoder starts decoding using the
858
*byteorder == -1: little endian
859
*byteorder == 0: native order
860
*byteorder == 1: big endian
862
In native mode, the first two bytes of the stream are checked for a
863
BOM mark. If found, the BOM mark is analysed, the byte order
864
adjusted and the BOM skipped. In the other modes, no BOM mark
865
interpretation is done. After completion, *byteorder is set to the
866
current byte order at the end of input data.
868
If byteorder is NULL, the codec starts in native order mode.
872
PyAPI_FUNC(PyObject*) PyUnicode_DecodeUTF16(
873
const char *string, /* UTF-16 encoded string */
874
Py_ssize_t length, /* size of string */
875
const char *errors, /* error handling */
876
int *byteorder /* pointer to byteorder to use
877
0=native;-1=LE,1=BE; updated on
881
PyAPI_FUNC(PyObject*) PyUnicode_DecodeUTF16Stateful(
882
const char *string, /* UTF-16 encoded string */
883
Py_ssize_t length, /* size of string */
884
const char *errors, /* error handling */
885
int *byteorder, /* pointer to byteorder to use
886
0=native;-1=LE,1=BE; updated on
888
Py_ssize_t *consumed /* bytes consumed */
891
/* Returns a Python string using the UTF-16 encoding in native byte
892
order. The string always starts with a BOM mark. */
894
PyAPI_FUNC(PyObject*) PyUnicode_AsUTF16String(
895
PyObject *unicode /* Unicode object */
898
/* Returns a Python string object holding the UTF-16 encoded value of
901
If byteorder is not 0, output is written according to the following
904
byteorder == -1: little endian
905
byteorder == 0: native byte order (writes a BOM mark)
906
byteorder == 1: big endian
908
If byteorder is 0, the output string will always start with the
909
Unicode BOM mark (U+FEFF). In the other two modes, no BOM mark is
912
Note that Py_UNICODE data is being interpreted as UTF-16 reduced to
913
UCS-2. This trick makes it possible to add full UTF-16 capabilities
914
at a later point without compromising the APIs.
918
PyAPI_FUNC(PyObject*) PyUnicode_EncodeUTF16(
919
const Py_UNICODE *data, /* Unicode char buffer */
920
Py_ssize_t length, /* number of Py_UNICODE chars to encode */
921
const char *errors, /* error handling */
922
int byteorder /* byteorder to use 0=BOM+native;-1=LE,1=BE */
925
/* --- Unicode-Escape Codecs ---------------------------------------------- */
927
PyAPI_FUNC(PyObject*) PyUnicode_DecodeUnicodeEscape(
928
const char *string, /* Unicode-Escape encoded string */
929
Py_ssize_t length, /* size of string */
930
const char *errors /* error handling */
933
PyAPI_FUNC(PyObject*) PyUnicode_AsUnicodeEscapeString(
934
PyObject *unicode /* Unicode object */
937
PyAPI_FUNC(PyObject*) PyUnicode_EncodeUnicodeEscape(
938
const Py_UNICODE *data, /* Unicode char buffer */
939
Py_ssize_t length /* Number of Py_UNICODE chars to encode */
942
/* --- Raw-Unicode-Escape Codecs ------------------------------------------ */
944
PyAPI_FUNC(PyObject*) PyUnicode_DecodeRawUnicodeEscape(
945
const char *string, /* Raw-Unicode-Escape encoded string */
946
Py_ssize_t length, /* size of string */
947
const char *errors /* error handling */
950
PyAPI_FUNC(PyObject*) PyUnicode_AsRawUnicodeEscapeString(
951
PyObject *unicode /* Unicode object */
954
PyAPI_FUNC(PyObject*) PyUnicode_EncodeRawUnicodeEscape(
955
const Py_UNICODE *data, /* Unicode char buffer */
956
Py_ssize_t length /* Number of Py_UNICODE chars to encode */
959
/* --- Unicode Internal Codec ---------------------------------------------
961
Only for internal use in _codecsmodule.c */
963
PyObject *_PyUnicode_DecodeUnicodeInternal(
969
/* --- Latin-1 Codecs -----------------------------------------------------
971
Note: Latin-1 corresponds to the first 256 Unicode ordinals.
975
PyAPI_FUNC(PyObject*) PyUnicode_DecodeLatin1(
976
const char *string, /* Latin-1 encoded string */
977
Py_ssize_t length, /* size of string */
978
const char *errors /* error handling */
981
PyAPI_FUNC(PyObject*) PyUnicode_AsLatin1String(
982
PyObject *unicode /* Unicode object */
985
PyAPI_FUNC(PyObject*) PyUnicode_EncodeLatin1(
986
const Py_UNICODE *data, /* Unicode char buffer */
987
Py_ssize_t length, /* Number of Py_UNICODE chars to encode */
988
const char *errors /* error handling */
991
/* --- ASCII Codecs -------------------------------------------------------
993
Only 7-bit ASCII data is excepted. All other codes generate errors.
997
PyAPI_FUNC(PyObject*) PyUnicode_DecodeASCII(
998
const char *string, /* ASCII encoded string */
999
Py_ssize_t length, /* size of string */
1000
const char *errors /* error handling */
1003
PyAPI_FUNC(PyObject*) PyUnicode_AsASCIIString(
1004
PyObject *unicode /* Unicode object */
1007
PyAPI_FUNC(PyObject*) PyUnicode_EncodeASCII(
1008
const Py_UNICODE *data, /* Unicode char buffer */
1009
Py_ssize_t length, /* Number of Py_UNICODE chars to encode */
1010
const char *errors /* error handling */
1013
/* --- Character Map Codecs -----------------------------------------------
1015
This codec uses mappings to encode and decode characters.
1017
Decoding mappings must map single string characters to single
1018
Unicode characters, integers (which are then interpreted as Unicode
1019
ordinals) or None (meaning "undefined mapping" and causing an
1022
Encoding mappings must map single Unicode characters to single
1023
string characters, integers (which are then interpreted as Latin-1
1024
ordinals) or None (meaning "undefined mapping" and causing an
1027
If a character lookup fails with a LookupError, the character is
1028
copied as-is meaning that its ordinal value will be interpreted as
1029
Unicode or Latin-1 ordinal resp. Because of this mappings only need
1030
to contain those mappings which map characters to different code
1035
PyAPI_FUNC(PyObject*) PyUnicode_DecodeCharmap(
1036
const char *string, /* Encoded string */
1037
Py_ssize_t length, /* size of string */
1038
PyObject *mapping, /* character mapping
1039
(char ordinal -> unicode ordinal) */
1040
const char *errors /* error handling */
1043
PyAPI_FUNC(PyObject*) PyUnicode_AsCharmapString(
1044
PyObject *unicode, /* Unicode object */
1045
PyObject *mapping /* character mapping
1046
(unicode ordinal -> char ordinal) */
1049
PyAPI_FUNC(PyObject*) PyUnicode_EncodeCharmap(
1050
const Py_UNICODE *data, /* Unicode char buffer */
1051
Py_ssize_t length, /* Number of Py_UNICODE chars to encode */
1052
PyObject *mapping, /* character mapping
1053
(unicode ordinal -> char ordinal) */
1054
const char *errors /* error handling */
1057
/* Translate a Py_UNICODE buffer of the given length by applying a
1058
character mapping table to it and return the resulting Unicode
1061
The mapping table must map Unicode ordinal integers to Unicode
1062
ordinal integers or None (causing deletion of the character).
1064
Mapping tables may be dictionaries or sequences. Unmapped character
1065
ordinals (ones which cause a LookupError) are left untouched and
1070
PyAPI_FUNC(PyObject *) PyUnicode_TranslateCharmap(
1071
const Py_UNICODE *data, /* Unicode char buffer */
1072
Py_ssize_t length, /* Number of Py_UNICODE chars to encode */
1073
PyObject *table, /* Translate table */
1074
const char *errors /* error handling */
1079
/* --- MBCS codecs for Windows -------------------------------------------- */
1081
PyAPI_FUNC(PyObject*) PyUnicode_DecodeMBCS(
1082
const char *string, /* MBCS encoded string */
1083
Py_ssize_t length, /* size of string */
1084
const char *errors /* error handling */
1087
PyAPI_FUNC(PyObject*) PyUnicode_DecodeMBCSStateful(
1088
const char *string, /* MBCS encoded string */
1089
Py_ssize_t length, /* size of string */
1090
const char *errors, /* error handling */
1091
Py_ssize_t *consumed /* bytes consumed */
1094
PyAPI_FUNC(PyObject*) PyUnicode_AsMBCSString(
1095
PyObject *unicode /* Unicode object */
1098
PyAPI_FUNC(PyObject*) PyUnicode_EncodeMBCS(
1099
const Py_UNICODE *data, /* Unicode char buffer */
1100
Py_ssize_t length, /* Number of Py_UNICODE chars to encode */
1101
const char *errors /* error handling */
1104
#endif /* MS_WIN32 */
1106
/* --- Decimal Encoder ---------------------------------------------------- */
1108
/* Takes a Unicode string holding a decimal value and writes it into
1109
an output buffer using standard ASCII digit codes.
1111
The output buffer has to provide at least length+1 bytes of storage
1112
area. The output string is 0-terminated.
1114
The encoder converts whitespace to ' ', decimal characters to their
1115
corresponding ASCII digit and all other Latin-1 characters except
1116
\0 as-is. Characters outside this range (Unicode ordinals 1-256)
1117
are treated as errors. This includes embedded NULL bytes.
1119
Error handling is defined by the errors argument:
1121
NULL or "strict": raise a ValueError
1122
"ignore": ignore the wrong characters (these are not copied to the
1124
"replace": replaces illegal characters with '?'
1126
Returns 0 on success, -1 on failure.
1130
PyAPI_FUNC(int) PyUnicode_EncodeDecimal(
1131
Py_UNICODE *s, /* Unicode buffer */
1132
Py_ssize_t length, /* Number of Py_UNICODE chars to encode */
1133
char *output, /* Output buffer; must have size >= length */
1134
const char *errors /* error handling */
1137
/* --- Methods & Slots ----------------------------------------------------
1139
These are capable of handling Unicode objects and strings on input
1140
(we refer to them as strings in the descriptions) and return
1141
Unicode objects or integers as apporpriate. */
1143
/* Concat two strings giving a new Unicode string. */
1145
PyAPI_FUNC(PyObject*) PyUnicode_Concat(
1146
PyObject *left, /* Left string */
1147
PyObject *right /* Right string */
1150
/* Split a string giving a list of Unicode strings.
1152
If sep is NULL, splitting will be done at all whitespace
1153
substrings. Otherwise, splits occur at the given separator.
1155
At most maxsplit splits will be done. If negative, no limit is set.
1157
Separators are not included in the resulting list.
1161
PyAPI_FUNC(PyObject*) PyUnicode_Split(
1162
PyObject *s, /* String to split */
1163
PyObject *sep, /* String separator */
1164
Py_ssize_t maxsplit /* Maxsplit count */
1167
/* Dito, but split at line breaks.
1169
CRLF is considered to be one line break. Line breaks are not
1170
included in the resulting list. */
1172
PyAPI_FUNC(PyObject*) PyUnicode_Splitlines(
1173
PyObject *s, /* String to split */
1174
int keepends /* If true, line end markers are included */
1177
/* Partition a string using a given separator. */
1179
PyAPI_FUNC(PyObject*) PyUnicode_Partition(
1180
PyObject *s, /* String to partition */
1181
PyObject *sep /* String separator */
1184
/* Partition a string using a given separator, searching from the end of the
1187
PyAPI_FUNC(PyObject*) PyUnicode_RPartition(
1188
PyObject *s, /* String to partition */
1189
PyObject *sep /* String separator */
1192
/* Split a string giving a list of Unicode strings.
1194
If sep is NULL, splitting will be done at all whitespace
1195
substrings. Otherwise, splits occur at the given separator.
1197
At most maxsplit splits will be done. But unlike PyUnicode_Split
1198
PyUnicode_RSplit splits from the end of the string. If negative,
1201
Separators are not included in the resulting list.
1205
PyAPI_FUNC(PyObject*) PyUnicode_RSplit(
1206
PyObject *s, /* String to split */
1207
PyObject *sep, /* String separator */
1208
Py_ssize_t maxsplit /* Maxsplit count */
1211
/* Translate a string by applying a character mapping table to it and
1212
return the resulting Unicode object.
1214
The mapping table must map Unicode ordinal integers to Unicode
1215
ordinal integers or None (causing deletion of the character).
1217
Mapping tables may be dictionaries or sequences. Unmapped character
1218
ordinals (ones which cause a LookupError) are left untouched and
1223
PyAPI_FUNC(PyObject *) PyUnicode_Translate(
1224
PyObject *str, /* String */
1225
PyObject *table, /* Translate table */
1226
const char *errors /* error handling */
1229
/* Join a sequence of strings using the given separator and return
1230
the resulting Unicode string. */
1232
PyAPI_FUNC(PyObject*) PyUnicode_Join(
1233
PyObject *separator, /* Separator string */
1234
PyObject *seq /* Sequence object */
1237
/* Return 1 if substr matches str[start:end] at the given tail end, 0
1240
PyAPI_FUNC(Py_ssize_t) PyUnicode_Tailmatch(
1241
PyObject *str, /* String */
1242
PyObject *substr, /* Prefix or Suffix string */
1243
Py_ssize_t start, /* Start index */
1244
Py_ssize_t end, /* Stop index */
1245
int direction /* Tail end: -1 prefix, +1 suffix */
1248
/* Return the first position of substr in str[start:end] using the
1249
given search direction or -1 if not found. -2 is returned in case
1250
an error occurred and an exception is set. */
1252
PyAPI_FUNC(Py_ssize_t) PyUnicode_Find(
1253
PyObject *str, /* String */
1254
PyObject *substr, /* Substring to find */
1255
Py_ssize_t start, /* Start index */
1256
Py_ssize_t end, /* Stop index */
1257
int direction /* Find direction: +1 forward, -1 backward */
1260
/* Count the number of occurrences of substr in str[start:end]. */
1262
PyAPI_FUNC(Py_ssize_t) PyUnicode_Count(
1263
PyObject *str, /* String */
1264
PyObject *substr, /* Substring to count */
1265
Py_ssize_t start, /* Start index */
1266
Py_ssize_t end /* Stop index */
1269
/* Replace at most maxcount occurrences of substr in str with replstr
1270
and return the resulting Unicode object. */
1272
PyAPI_FUNC(PyObject *) PyUnicode_Replace(
1273
PyObject *str, /* String */
1274
PyObject *substr, /* Substring to find */
1275
PyObject *replstr, /* Substring to replace */
1276
Py_ssize_t maxcount /* Max. number of replacements to apply;
1280
/* Compare two strings and return -1, 0, 1 for less than, equal,
1281
greater than resp. */
1283
PyAPI_FUNC(int) PyUnicode_Compare(
1284
PyObject *left, /* Left string */
1285
PyObject *right /* Right string */
1288
/* Rich compare two strings and return one of the following:
1290
- NULL in case an exception was raised
1291
- Py_True or Py_False for successfuly comparisons
1292
- Py_NotImplemented in case the type combination is unknown
1294
Note that Py_EQ and Py_NE comparisons can cause a UnicodeWarning in
1295
case the conversion of the arguments to Unicode fails with a
1298
Possible values for op:
1300
Py_GT, Py_GE, Py_EQ, Py_NE, Py_LT, Py_LE
1304
PyAPI_FUNC(PyObject *) PyUnicode_RichCompare(
1305
PyObject *left, /* Left string */
1306
PyObject *right, /* Right string */
1307
int op /* Operation: Py_EQ, Py_NE, Py_GT, etc. */
1310
/* Apply an argument tuple or dictionary to a format string and return
1311
the resulting Unicode string. */
1313
PyAPI_FUNC(PyObject *) PyUnicode_Format(
1314
PyObject *format, /* Format string */
1315
PyObject *args /* Argument tuple or dictionary */
1318
/* Checks whether element is contained in container and return 1/0
1321
element has to coerce to a one element Unicode string. -1 is
1322
returned in case of an error. */
1324
PyAPI_FUNC(int) PyUnicode_Contains(
1325
PyObject *container, /* Container string */
1326
PyObject *element /* Element string */
1329
/* Externally visible for str.strip(unicode) */
1330
PyAPI_FUNC(PyObject *) _PyUnicode_XStrip(
1331
PyUnicodeObject *self,
1336
/* === Characters Type APIs =============================================== */
1338
/* Helper array used by Py_UNICODE_ISSPACE(). */
1340
PyAPI_DATA(const unsigned char) _Py_ascii_whitespace[];
1342
/* These should not be used directly. Use the Py_UNICODE_IS* and
1343
Py_UNICODE_TO* macros instead.
1345
These APIs are implemented in Objects/unicodectype.c.
1349
PyAPI_FUNC(int) _PyUnicode_IsLowercase(
1350
Py_UNICODE ch /* Unicode character */
1353
PyAPI_FUNC(int) _PyUnicode_IsUppercase(
1354
Py_UNICODE ch /* Unicode character */
1357
PyAPI_FUNC(int) _PyUnicode_IsTitlecase(
1358
Py_UNICODE ch /* Unicode character */
1361
PyAPI_FUNC(int) _PyUnicode_IsWhitespace(
1362
const Py_UNICODE ch /* Unicode character */
1365
PyAPI_FUNC(int) _PyUnicode_IsLinebreak(
1366
const Py_UNICODE ch /* Unicode character */
1369
PyAPI_FUNC(Py_UNICODE) _PyUnicode_ToLowercase(
1370
Py_UNICODE ch /* Unicode character */
1373
PyAPI_FUNC(Py_UNICODE) _PyUnicode_ToUppercase(
1374
Py_UNICODE ch /* Unicode character */
1377
PyAPI_FUNC(Py_UNICODE) _PyUnicode_ToTitlecase(
1378
Py_UNICODE ch /* Unicode character */
1381
PyAPI_FUNC(int) _PyUnicode_ToDecimalDigit(
1382
Py_UNICODE ch /* Unicode character */
1385
PyAPI_FUNC(int) _PyUnicode_ToDigit(
1386
Py_UNICODE ch /* Unicode character */
1389
PyAPI_FUNC(double) _PyUnicode_ToNumeric(
1390
Py_UNICODE ch /* Unicode character */
1393
PyAPI_FUNC(int) _PyUnicode_IsDecimalDigit(
1394
Py_UNICODE ch /* Unicode character */
1397
PyAPI_FUNC(int) _PyUnicode_IsDigit(
1398
Py_UNICODE ch /* Unicode character */
1401
PyAPI_FUNC(int) _PyUnicode_IsNumeric(
1402
Py_UNICODE ch /* Unicode character */
1405
PyAPI_FUNC(int) _PyUnicode_IsAlpha(
1406
Py_UNICODE ch /* Unicode character */
1412
#endif /* Py_USING_UNICODE */
1413
#endif /* !Py_UNICODEOBJECT_H */
added
removed
python/include/unicodeobject.h
