1
# Copyright (C) 2004-2006 Python Software Foundation
2
# Authors: Baxter, Wouters and Warsaw
3
# Contact: email-sig@python.org
5
"""FeedParser - An email feed parser.
7
The feed parser implements an interface for incrementally parsing an email
8
message, line by line. This has advantages for certain applications, such as
9
those reading email messages off a socket.
11
FeedParser.feed() is the primary interface for pushing new data into the
12
parser. It returns when there's nothing more it can do with the available
13
data. When you have no more data to push into the parser, call .close().
14
This completes the parsing and returns the root message object.
16
The other advantage of this parser is that it will never throw a parsing
17
exception. Instead, when it finds something unexpected, it adds a 'defect' to
18
the current message. Defects are just instances that live on the message
19
object's .defects attribute.
22
__all__ = ['FeedParser']
26
from email import errors
27
from email import message
29
NLCRE = re.compile('\r\n|\r|\n')
30
NLCRE_bol = re.compile('(\r\n|\r|\n)')
31
NLCRE_eol = re.compile('(\r\n|\r|\n)$')
32
NLCRE_crack = re.compile('(\r\n|\r|\n)')
33
# RFC 2822 $3.6.8 Optional fields. ftext is %d33-57 / %d59-126, Any character
34
# except controls, SP, and ":".
35
headerRE = re.compile(r'^(From |[\041-\071\073-\176]{1,}:|[\t ])')
39
NeedMoreData = object()
43
class BufferedSubFile(object):
44
"""A file-ish object that can have new data loaded into it.
46
You can also push and pop line-matching predicates onto a stack. When the
47
current predicate matches the current line, a false EOF response
48
(i.e. empty string) is returned instead. This lets the parser adhere to a
49
simple abstraction -- it parses until EOF closes the current message.
52
# The last partial line pushed into this object.
54
# The list of full, pushed lines, in reverse order
56
# The stack of false-EOF checking predicates.
58
# A flag indicating whether the file has been closed or not.
61
def push_eof_matcher(self, pred):
62
self._eofstack.append(pred)
64
def pop_eof_matcher(self):
65
return self._eofstack.pop()
68
# Don't forget any trailing partial line.
69
self._lines.append(self._partial)
78
# Pop the line off the stack and see if it matches the current
79
# false-EOF predicate.
80
line = self._lines.pop()
81
# RFC 2046, section 5.1.2 requires us to recognize outer level
82
# boundaries at any level of inner nesting. Do this, but be sure it's
83
# in the order of most to least nested.
84
for ateof in self._eofstack[::-1]:
86
# We're at the false EOF. But push the last line back first.
87
self._lines.append(line)
91
def unreadline(self, line):
92
# Let the consumer push a line back into the buffer.
93
assert line is not NeedMoreData
94
self._lines.append(line)
97
"""Push some new data into this object."""
98
# Handle any previous leftovers
99
data, self._partial = self._partial + data, ''
100
# Crack into lines, but preserve the newlines on the end of each
101
parts = NLCRE_crack.split(data)
102
# The *ahem* interesting behaviour of re.split when supplied grouping
103
# parentheses is that the last element of the resulting list is the
104
# data after the final RE. In the case of a NL/CR terminated string,
105
# this is the empty string.
106
self._partial = parts.pop()
107
# parts is a list of strings, alternating between the line contents
108
# and the eol character(s). Gather up a list of lines after
109
# re-attaching the newlines.
111
for i in range(len(parts) // 2):
112
lines.append(parts[i*2] + parts[i*2+1])
113
self.pushlines(lines)
115
def pushlines(self, lines):
116
# Reverse and insert at the front of the lines.
117
self._lines[:0] = lines[::-1]
126
line = self.readline()
134
"""A feed-style parser of email."""
136
def __init__(self, _factory=message.Message):
137
"""_factory is called with no arguments to create a new message obj"""
138
self._factory = _factory
139
self._input = BufferedSubFile()
141
self._parse = self._parsegen().__next__
144
self._headersonly = False
146
# Non-public interface for supporting Parser's headersonly flag
147
def _set_headersonly(self):
148
self._headersonly = True
150
def feed(self, data):
151
"""Push more data into the parser."""
152
self._input.push(data)
155
def _call_parse(self):
158
except StopIteration:
162
"""Parse all remaining data and return the root message object."""
165
root = self._pop_message()
166
assert not self._msgstack
167
# Look for final set of defects
168
if root.get_content_maintype() == 'multipart' \
169
and not root.is_multipart():
170
root.defects.append(errors.MultipartInvariantViolationDefect())
173
def _new_message(self):
174
msg = self._factory()
175
if self._cur and self._cur.get_content_type() == 'multipart/digest':
176
msg.set_default_type('message/rfc822')
178
self._msgstack[-1].attach(msg)
179
self._msgstack.append(msg)
183
def _pop_message(self):
184
retval = self._msgstack.pop()
186
self._cur = self._msgstack[-1]
192
# Create a new message and start by parsing headers.
195
# Collect the headers, searching for a line that doesn't match the RFC
196
# 2822 header or continuation pattern (including an empty line).
197
for line in self._input:
198
if line is NeedMoreData:
201
if not headerRE.match(line):
202
# If we saw the RFC defined header/body separator
203
# (i.e. newline), just throw it away. Otherwise the line is
204
# part of the body so push it back.
205
if not NLCRE.match(line):
206
self._input.unreadline(line)
209
# Done with the headers, so parse them and figure out what we're
210
# supposed to see in the body of the message.
211
self._parse_headers(headers)
212
# Headers-only parsing is a backwards compatibility hack, which was
213
# necessary in the older parser, which could throw errors. All
214
# remaining lines in the input are thrown into the message body.
215
if self._headersonly:
218
line = self._input.readline()
219
if line is NeedMoreData:
225
self._cur.set_payload(EMPTYSTRING.join(lines))
227
if self._cur.get_content_type() == 'message/delivery-status':
228
# message/delivery-status contains blocks of headers separated by
229
# a blank line. We'll represent each header block as a separate
230
# nested message object, but the processing is a bit different
231
# than standard message/* types because there is no body for the
232
# nested messages. A blank line separates the subparts.
234
self._input.push_eof_matcher(NLCRE.match)
235
for retval in self._parsegen():
236
if retval is NeedMoreData:
240
msg = self._pop_message()
241
# We need to pop the EOF matcher in order to tell if we're at
242
# the end of the current file, not the end of the last block
243
# of message headers.
244
self._input.pop_eof_matcher()
245
# The input stream must be sitting at the newline or at the
246
# EOF. We want to see if we're at the end of this subpart, so
247
# first consume the blank line, then test the next line to see
248
# if we're at this subpart's EOF.
250
line = self._input.readline()
251
if line is NeedMoreData:
256
line = self._input.readline()
257
if line is NeedMoreData:
263
# Not at EOF so this is a line we're going to need.
264
self._input.unreadline(line)
266
if self._cur.get_content_maintype() == 'message':
267
# The message claims to be a message/* type, then what follows is
268
# another RFC 2822 message.
269
for retval in self._parsegen():
270
if retval is NeedMoreData:
276
if self._cur.get_content_maintype() == 'multipart':
277
boundary = self._cur.get_boundary()
279
# The message /claims/ to be a multipart but it has not
280
# defined a boundary. That's a problem which we'll handle by
281
# reading everything until the EOF and marking the message as
283
self._cur.defects.append(errors.NoBoundaryInMultipartDefect())
285
for line in self._input:
286
if line is NeedMoreData:
290
self._cur.set_payload(EMPTYSTRING.join(lines))
292
# Create a line match predicate which matches the inter-part
293
# boundary as well as the end-of-multipart boundary. Don't push
294
# this onto the input stream until we've scanned past the
296
separator = '--' + boundary
297
boundaryre = re.compile(
298
'(?P<sep>' + re.escape(separator) +
299
r')(?P<end>--)?(?P<ws>[ \t]*)(?P<linesep>\r\n|\r|\n)?$')
300
capturing_preamble = True
304
line = self._input.readline()
305
if line is NeedMoreData:
310
mo = boundaryre.match(line)
312
# If we're looking at the end boundary, we're done with
313
# this multipart. If there was a newline at the end of
314
# the closing boundary, then we need to initialize the
315
# epilogue with the empty string (see below).
317
linesep = mo.group('linesep')
319
# We saw an inter-part boundary. Were we in the preamble?
320
if capturing_preamble:
322
# According to RFC 2046, the last newline belongs
324
lastline = preamble[-1]
325
eolmo = NLCRE_eol.search(lastline)
327
preamble[-1] = lastline[:-len(eolmo.group(0))]
328
self._cur.preamble = EMPTYSTRING.join(preamble)
329
capturing_preamble = False
330
self._input.unreadline(line)
332
# We saw a boundary separating two parts. Consume any
333
# multiple boundary lines that may be following. Our
334
# interpretation of RFC 2046 BNF grammar does not produce
335
# body parts within such double boundaries.
337
line = self._input.readline()
338
if line is NeedMoreData:
341
mo = boundaryre.match(line)
343
self._input.unreadline(line)
345
# Recurse to parse this subpart; the input stream points
346
# at the subpart's first line.
347
self._input.push_eof_matcher(boundaryre.match)
348
for retval in self._parsegen():
349
if retval is NeedMoreData:
353
# Because of RFC 2046, the newline preceding the boundary
354
# separator actually belongs to the boundary, not the
355
# previous subpart's payload (or epilogue if the previous
356
# part is a multipart).
357
if self._last.get_content_maintype() == 'multipart':
358
epilogue = self._last.epilogue
360
self._last.epilogue = None
361
elif epilogue is not None:
362
mo = NLCRE_eol.search(epilogue)
364
end = len(mo.group(0))
365
self._last.epilogue = epilogue[:-end]
367
payload = self._last.get_payload()
368
if isinstance(payload, str):
369
mo = NLCRE_eol.search(payload)
371
payload = payload[:-len(mo.group(0))]
372
self._last.set_payload(payload)
373
self._input.pop_eof_matcher()
375
# Set the multipart up for newline cleansing, which will
376
# happen if we're in a nested multipart.
377
self._last = self._cur
379
# I think we must be in the preamble
380
assert capturing_preamble
381
preamble.append(line)
382
# We've seen either the EOF or the end boundary. If we're still
383
# capturing the preamble, we never saw the start boundary. Note
384
# that as a defect and store the captured text as the payload.
385
# Everything from here to the EOF is epilogue.
386
if capturing_preamble:
387
self._cur.defects.append(errors.StartBoundaryNotFoundDefect())
388
self._cur.set_payload(EMPTYSTRING.join(preamble))
390
for line in self._input:
391
if line is NeedMoreData:
394
self._cur.epilogue = EMPTYSTRING.join(epilogue)
396
# If the end boundary ended in a newline, we'll need to make sure
397
# the epilogue isn't None
402
for line in self._input:
403
if line is NeedMoreData:
406
epilogue.append(line)
407
# Any CRLF at the front of the epilogue is not technically part of
408
# the epilogue. Also, watch out for an empty string epilogue,
409
# which means a single newline.
411
firstline = epilogue[0]
412
bolmo = NLCRE_bol.match(firstline)
414
epilogue[0] = firstline[len(bolmo.group(0)):]
415
self._cur.epilogue = EMPTYSTRING.join(epilogue)
417
# Otherwise, it's some non-multipart type, so the entire rest of the
418
# file contents becomes the payload.
420
for line in self._input:
421
if line is NeedMoreData:
425
self._cur.set_payload(EMPTYSTRING.join(lines))
427
def _parse_headers(self, lines):
428
# Passed a list of lines that make up the headers for the current msg
431
for lineno, line in enumerate(lines):
432
# Check for continuation
435
# The first line of the headers was a continuation. This
436
# is illegal, so let's note the defect, store the illegal
437
# line, and ignore it for purposes of headers.
438
defect = errors.FirstHeaderLineIsContinuationDefect(line)
439
self._cur.defects.append(defect)
441
lastvalue.append(line)
444
# XXX reconsider the joining of folded lines
445
lhdr = EMPTYSTRING.join(lastvalue)[:-1].rstrip('\r\n')
446
self._cur[lastheader] = lhdr
447
lastheader, lastvalue = '', []
448
# Check for envelope header, i.e. unix-from
449
if line.startswith('From '):
451
# Strip off the trailing newline
452
mo = NLCRE_eol.search(line)
454
line = line[:-len(mo.group(0))]
455
self._cur.set_unixfrom(line)
457
elif lineno == len(lines) - 1:
458
# Something looking like a unix-from at the end - it's
459
# probably the first line of the body, so push back the
461
self._input.unreadline(line)
464
# Weirdly placed unix-from line. Note this as a defect
466
defect = errors.MisplacedEnvelopeHeaderDefect(line)
467
self._cur.defects.append(defect)
469
# Split the line on the colon separating field name from value.
472
defect = errors.MalformedHeaderDefect(line)
473
self._cur.defects.append(defect)
475
lastheader = line[:i]
476
lastvalue = [line[i+1:].lstrip()]
477
# Done with all the lines, so handle the last header.
479
# XXX reconsider the joining of folded lines
480
self._cur[lastheader] = EMPTYSTRING.join(lastvalue).rstrip('\r\n')
added
removed
Lib/email/feedparser.py
