1
# Copyright (C) 2001-2007 Python Software Foundation
2
# Author: Ben Gertzfield, Barry Warsaw
3
# Contact: email-sig@python.org
12
from functools import partial
14
import email.base64mime
15
import email.quoprimime
17
from email import errors
18
from email.encoders import encode_7or8bit
22
# Flags for types of header encodings
23
QP = 1 # Quoted-Printable
25
SHORTEST = 3 # the shorter of QP and base64, but only for headers
27
# In "=?charset?q?hello_world?=", the =?, ?q?, and ?= add up to 7
28
RFC2047_CHROME_LEN = 7
30
DEFAULT_CHARSET = 'us-ascii'
37
# input header enc body enc output conv
38
'iso-8859-1': (QP, QP, None),
39
'iso-8859-2': (QP, QP, None),
40
'iso-8859-3': (QP, QP, None),
41
'iso-8859-4': (QP, QP, None),
42
# iso-8859-5 is Cyrillic, and not especially used
43
# iso-8859-6 is Arabic, also not particularly used
44
# iso-8859-7 is Greek, QP will not make it readable
45
# iso-8859-8 is Hebrew, QP will not make it readable
46
'iso-8859-9': (QP, QP, None),
47
'iso-8859-10': (QP, QP, None),
48
# iso-8859-11 is Thai, QP will not make it readable
49
'iso-8859-13': (QP, QP, None),
50
'iso-8859-14': (QP, QP, None),
51
'iso-8859-15': (QP, QP, None),
52
'iso-8859-16': (QP, QP, None),
53
'windows-1252':(QP, QP, None),
54
'viscii': (QP, QP, None),
55
'us-ascii': (None, None, None),
56
'big5': (BASE64, BASE64, None),
57
'gb2312': (BASE64, BASE64, None),
58
'euc-jp': (BASE64, None, 'iso-2022-jp'),
59
'shift_jis': (BASE64, None, 'iso-2022-jp'),
60
'iso-2022-jp': (BASE64, None, None),
61
'koi8-r': (BASE64, BASE64, None),
62
'utf-8': (SHORTEST, BASE64, 'utf-8'),
65
# Aliases for other commonly-used names for character sets. Map
66
# them to the real ones used in email.
68
'latin_1': 'iso-8859-1',
69
'latin-1': 'iso-8859-1',
70
'latin_2': 'iso-8859-2',
71
'latin-2': 'iso-8859-2',
72
'latin_3': 'iso-8859-3',
73
'latin-3': 'iso-8859-3',
74
'latin_4': 'iso-8859-4',
75
'latin-4': 'iso-8859-4',
76
'latin_5': 'iso-8859-9',
77
'latin-5': 'iso-8859-9',
78
'latin_6': 'iso-8859-10',
79
'latin-6': 'iso-8859-10',
80
'latin_7': 'iso-8859-13',
81
'latin-7': 'iso-8859-13',
82
'latin_8': 'iso-8859-14',
83
'latin-8': 'iso-8859-14',
84
'latin_9': 'iso-8859-15',
85
'latin-9': 'iso-8859-15',
86
'latin_10':'iso-8859-16',
87
'latin-10':'iso-8859-16',
88
'cp949': 'ks_c_5601-1987',
95
# Map charsets to their Unicode codec strings.
97
'gb2312': 'eucgb2312_cn',
99
# Hack: We don't want *any* conversion for stuff marked us-ascii, as all
100
# sorts of garbage might be sent to us in the guise of 7-bit us-ascii.
101
# Let that stuff pass through without conversion to/from Unicode.
107
# Convenience functions for extending the above mappings
108
def add_charset(charset, header_enc=None, body_enc=None, output_charset=None):
109
"""Add character set properties to the global registry.
111
charset is the input character set, and must be the canonical name of a
114
Optional header_enc and body_enc is either Charset.QP for
115
quoted-printable, Charset.BASE64 for base64 encoding, Charset.SHORTEST for
116
the shortest of qp or base64 encoding, or None for no encoding. SHORTEST
117
is only valid for header_enc. It describes how message headers and
118
message bodies in the input charset are to be encoded. Default is no
121
Optional output_charset is the character set that the output should be
122
in. Conversions will proceed from input charset, to Unicode, to the
123
output charset when the method Charset.convert() is called. The default
124
is to output in the same character set as the input.
126
Both input_charset and output_charset must have Unicode codec entries in
127
the module's charset-to-codec mapping; use add_codec(charset, codecname)
128
to add codecs the module does not know about. See the codecs module's
129
documentation for more information.
131
if body_enc == SHORTEST:
132
raise ValueError('SHORTEST not allowed for body_enc')
133
CHARSETS[charset] = (header_enc, body_enc, output_charset)
136
def add_alias(alias, canonical):
137
"""Add a character set alias.
139
alias is the alias name, e.g. latin-1
140
canonical is the character set's canonical name, e.g. iso-8859-1
142
ALIASES[alias] = canonical
145
def add_codec(charset, codecname):
146
"""Add a codec that map characters in the given charset to/from Unicode.
148
charset is the canonical name of a character set. codecname is the name
149
of a Python codec, as appropriate for the second argument to the unicode()
150
built-in, or to the encode() method of a Unicode string.
152
CODEC_MAP[charset] = codecname
157
"""Map character sets to their email properties.
159
This class provides information about the requirements imposed on email
160
for a specific character set. It also provides convenience routines for
161
converting between character sets, given the availability of the
162
applicable codecs. Given a character set, it will do its best to provide
163
information on how to use that character set in an email in an
166
Certain character sets must be encoded with quoted-printable or base64
167
when used in email headers or bodies. Certain character sets must be
168
converted outright, and are not allowed in email. Instances of this
169
module expose the following information about a character set:
171
input_charset: The initial character set specified. Common aliases
172
are converted to their `official' email names (e.g. latin_1
173
is converted to iso-8859-1). Defaults to 7-bit us-ascii.
175
header_encoding: If the character set must be encoded before it can be
176
used in an email header, this attribute will be set to
177
Charset.QP (for quoted-printable), Charset.BASE64 (for
178
base64 encoding), or Charset.SHORTEST for the shortest of
179
QP or BASE64 encoding. Otherwise, it will be None.
181
body_encoding: Same as header_encoding, but describes the encoding for the
182
mail message's body, which indeed may be different than the
183
header encoding. Charset.SHORTEST is not allowed for
186
output_charset: Some character sets must be converted before the can be
187
used in email headers or bodies. If the input_charset is
188
one of them, this attribute will contain the name of the
189
charset output will be converted to. Otherwise, it will
192
input_codec: The name of the Python codec used to convert the
193
input_charset to Unicode. If no conversion codec is
194
necessary, this attribute will be None.
196
output_codec: The name of the Python codec used to convert Unicode
197
to the output_charset. If no conversion codec is necessary,
198
this attribute will have the same value as the input_codec.
200
def __init__(self, input_charset=DEFAULT_CHARSET):
201
# RFC 2046, $4.1.2 says charsets are not case sensitive. We coerce to
202
# unicode because its .lower() is locale insensitive. If the argument
203
# is already a unicode, we leave it at that, but ensure that the
204
# charset is ASCII, as the standard (RFC XXX) requires.
206
if isinstance(input_charset, str):
207
input_charset.encode('ascii')
209
input_charset = str(input_charset, 'ascii')
211
raise errors.CharsetError(input_charset)
212
input_charset = input_charset.lower()
213
# Set the input charset after filtering through the aliases
214
self.input_charset = ALIASES.get(input_charset, input_charset)
215
# We can try to guess which encoding and conversion to use by the
216
# charset_map dictionary. Try that first, but let the user override
218
henc, benc, conv = CHARSETS.get(self.input_charset,
219
(SHORTEST, BASE64, None))
221
conv = self.input_charset
222
# Set the attributes, allowing the arguments to override the default.
223
self.header_encoding = henc
224
self.body_encoding = benc
225
self.output_charset = ALIASES.get(conv, conv)
226
# Now set the codecs. If one isn't defined for input_charset,
227
# guess and try a Unicode codec with the same name as input_codec.
228
self.input_codec = CODEC_MAP.get(self.input_charset,
230
self.output_codec = CODEC_MAP.get(self.output_charset,
234
return self.input_charset.lower()
238
def __eq__(self, other):
239
return str(self) == str(other).lower()
241
def __ne__(self, other):
242
return not self.__eq__(other)
244
def get_body_encoding(self):
245
"""Return the content-transfer-encoding used for body encoding.
247
This is either the string `quoted-printable' or `base64' depending on
248
the encoding used, or it is a function in which case you should call
249
the function with a single argument, the Message object being
250
encoded. The function should then set the Content-Transfer-Encoding
251
header itself to whatever is appropriate.
253
Returns "quoted-printable" if self.body_encoding is QP.
254
Returns "base64" if self.body_encoding is BASE64.
255
Returns "7bit" otherwise.
257
assert self.body_encoding != SHORTEST
258
if self.body_encoding == QP:
259
return 'quoted-printable'
260
elif self.body_encoding == BASE64:
263
return encode_7or8bit
265
def get_output_charset(self):
266
"""Return the output character set.
268
This is self.output_charset if that is not None, otherwise it is
271
return self.output_charset or self.input_charset
273
def header_encode(self, string):
274
"""Header-encode a string by converting it first to bytes.
276
The type of encoding (base64 or quoted-printable) will be based on
277
this charset's `header_encoding`.
279
:param string: A unicode string for the header. It must be possible
280
to encode this string to bytes using the character set's
282
:return: The encoded string, with RFC 2047 chrome.
284
codec = self.output_codec or 'us-ascii'
285
charset = self.get_output_charset()
286
header_bytes = string.encode(codec)
287
# 7bit/8bit encodings return the string unchanged (modulo conversions)
288
encoder_module = self._get_encoder(header_bytes)
289
if encoder_module is None:
291
return encoder_module.header_encode(header_bytes, codec)
293
def header_encode_lines(self, string, maxlengths):
294
"""Header-encode a string by converting it first to bytes.
296
This is similar to `header_encode()` except that the string is fit
297
into maximum line lengths as given by the arguments.
299
:param string: A unicode string for the header. It must be possible
300
to encode this string to bytes using the character set's
302
:param maxlengths: Maximum line length iterator. Each element
303
returned from this iterator will provide the next maximum line
304
length. This parameter is used as an argument to built-in next()
305
and should never be exhausted. The maximum line lengths should
306
not count the RFC 2047 chrome. These line lengths are only a
307
hint; the splitter does the best it can.
308
:param firstmaxlen: The maximum line length of the first line. If
309
None (the default), then `maxlen` is used for the first line.
310
:return: Lines of encoded strings, each with RFC 2047 chrome.
312
# See which encoding we should use.
313
codec = self.output_codec or 'us-ascii'
314
header_bytes = string.encode(codec)
315
encoder_module = self._get_encoder(header_bytes)
316
encoder = partial(encoder_module.header_encode, charset=str(self))
317
# Calculate the number of characters that the RFC 2047 chrome will
318
# contribute to each line.
319
charset = self.get_output_charset()
320
extra = len(charset) + RFC2047_CHROME_LEN
321
# Now comes the hard part. We must encode bytes but we can't split on
322
# bytes because some character sets are variable length and each
323
# encoded word must stand on its own. So the problem is you have to
324
# encode to bytes to figure out this word's length, but you must split
325
# on characters. This causes two problems: first, we don't know how
326
# many octets a specific substring of unicode characters will get
327
# encoded to, and second, we don't know how many ASCII characters
328
# those octets will get encoded to. Unless we try it. Which seems
329
# inefficient. In the interest of being correct rather than fast (and
330
# in the hope that there will be few encoded headers in any such
331
# message), brute force it. :(
334
maxlen = next(maxlengths) - extra
335
for character in string:
336
current_line.append(character)
337
this_line = EMPTYSTRING.join(current_line)
338
length = encoder_module.header_length(this_line.encode(charset))
340
# This last character doesn't fit so pop it off.
342
# Does nothing fit on the first line?
343
if not lines and not current_line:
346
separator = (' ' if lines else '')
347
joined_line = EMPTYSTRING.join(current_line)
348
header_bytes = joined_line.encode(codec)
349
lines.append(encoder(header_bytes))
350
current_line = [character]
351
maxlen = next(maxlengths) - extra
352
joined_line = EMPTYSTRING.join(current_line)
353
header_bytes = joined_line.encode(codec)
354
lines.append(encoder(header_bytes))
357
def _get_encoder(self, header_bytes):
358
if self.header_encoding == BASE64:
359
return email.base64mime
360
elif self.header_encoding == QP:
361
return email.quoprimime
362
elif self.header_encoding == SHORTEST:
363
len64 = email.base64mime.header_length(header_bytes)
364
lenqp = email.quoprimime.header_length(header_bytes)
366
return email.base64mime
368
return email.quoprimime
372
def body_encode(self, string):
373
"""Body-encode a string by converting it first to bytes.
375
The type of encoding (base64 or quoted-printable) will be based on
378
# 7bit/8bit encodings return the string unchanged (module conversions)
379
if self.body_encoding is BASE64:
380
return email.base64mime.body_encode(string)
381
elif self.body_encoding is QP:
382
return email.quoprimime.body_encode(string)
added
removed
Lib/email/charset.py
