1
# Copyright (C) 2001,2002 by the Free Software Foundation, Inc.
3
# This program is free software; you can redistribute it and/or
4
# modify it under the terms of the GNU General Public License
5
# as published by the Free Software Foundation; either version 2
6
# of the License, or (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17
"""Cleanse a message for archiving.
28
from cStringIO import StringIO
29
from types import IntType
31
from email.Utils import parsedate
32
from email.Parser import HeaderParser
33
from email.Generator import Generator
35
from Mailman import mm_cfg
36
from Mailman import Utils
37
from Mailman import LockFile
38
from Mailman import Message
39
from Mailman.Errors import DiscardMessage
40
from Mailman.i18n import _
41
from Mailman.Logging.Syslog import syslog
43
# Path characters for common platforms
44
pre = re.compile(r'[/\\:]')
45
# All other characters to strip out of Content-Disposition: filenames
46
# (essentially anything that isn't an alphanum, dot, slash, or underscore.
47
sre = re.compile(r'[^-\w.]')
48
# Regexp to strip out leading dots
49
dre = re.compile(r'^\.*')
56
# We're using a subclass of the standard Generator because we want to suppress
57
# headers in the subparts of multiparts. We use a hack -- the ctor argument
58
# skipheaders to accomplish this. It's set to true for the outer Message
59
# object, but false for all internal objects. We recognize that
60
# sub-Generators will get created passing only mangle_from_ and maxheaderlen
63
# This isn't perfect because we still get stuff like the multipart boundaries,
64
# but see below for how we corrupt that to our nefarious goals.
65
class ScrubberGenerator(Generator):
66
def __init__(self, outfp, mangle_from_=1, maxheaderlen=78, skipheaders=1):
67
Generator.__init__(self, outfp, mangle_from_=0)
68
self.__skipheaders = skipheaders
70
def _write_headers(self, msg):
71
if not self.__skipheaders:
72
Generator._write_headers(self, msg)
75
def safe_strftime(fmt, floatsecs):
77
return time.strftime(fmt, floatsecs)
82
def calculate_attachments_dir(mlist, msg, msgdata):
83
# Calculate the directory that attachments for this message will go
84
# under. To avoid inode limitations, the scheme will be:
85
# archives/private/<listname>/attachments/YYYYMMDD/<msgid-hash>/<files>
86
# Start by calculating the date-based and msgid-hash components.
88
datestr = msg.get('Date')
90
now = parsedate(datestr)
92
now = time.gmtime(msgdata.get('received_time', time.time()))
93
datedir = safe_strftime(fmt, now)
95
datestr = msgdata.get('X-List-Received-Date')
97
datedir = safe_strftime(fmt, datestr)
99
# What next? Unixfrom, I guess.
100
parts = msg.get_unixfrom().split()
102
month = {'Jan':1, 'Feb':2, 'Mar':3, 'Apr':4, 'May':5, 'Jun':6,
103
'Jul':7, 'Aug':8, 'Sep':9, 'Oct':10, 'Nov':11, 'Dec':12,
107
except (IndexError, ValueError):
108
# Best we can do I think
109
month = day = year = 0
110
datedir = '%04d%02d%02d' % (year, month, day)
112
# As for the msgid hash, we'll base this part on the Message-ID: so that
113
# all attachments for the same message end up in the same directory (we'll
114
# uniquify the filenames in that directory as needed). We use the first 2
115
# and last 2 bytes of the SHA1 hash of the message id as the basis of the
116
# directory name. Clashes here don't really matter too much, and that
117
# still gives us a 32-bit space to work with.
118
msgid = msg['message-id']
120
msgid = msg['Message-ID'] = Utils.unique_message_id(mlist)
121
# We assume that the message id actually /is/ unique!
122
digest = sha.new(msgid).hexdigest()
123
return os.path.join('attachments', datedir, digest[:4] + digest[-4:])
127
def process(mlist, msg, msgdata=None):
128
sanitize = mm_cfg.ARCHIVE_HTML_SANITIZER
132
dir = calculate_attachments_dir(mlist, msg, msgdata)
134
# Now walk over all subparts of this message and scrub out various types
135
for part in msg.walk():
136
ctype = part.get_type(part.get_default_type())
137
# If the part is text/plain, we leave it alone
138
if ctype == 'text/plain':
139
# We need to choose a charset for the scrubbed message, so we'll
140
# arbitrarily pick the charset of the first text/plain part in the
143
charset = part.get_content_charset(charset)
144
elif ctype == 'text/html' and isinstance(sanitize, IntType):
148
part.set_payload(_('HTML attachment scrubbed and removed'))
149
part.set_type('text/plain')
151
# By leaving it alone, Pipermail will automatically escape it
154
# Pull it out as an attachment but leave it unescaped. This
155
# is dangerous, but perhaps useful for heavily moderated
157
omask = os.umask(002)
159
url = save_attachment(mlist, part, dir, filter_html=0)
162
part.set_payload(_("""\
163
An HTML attachment was scrubbed...
166
part.set_type('text/plain')
168
# HTML-escape it and store it as an attachment, but make it
169
# look a /little/ bit prettier. :(
170
payload = Utils.websafe(part.get_payload(decode=1))
171
# For whitespace in the margin, change spaces into
172
# non-breaking spaces, and tabs into 8 of those. Then use a
173
# mono-space font. Still looks hideous to me, but then I'd
174
# just as soon discard them.
176
return s.replace(' ', ' ').replace('\t', ' '*8)
177
lines = [doreplace(s) for s in payload.split('\n')]
178
payload = '<tt>\n' + BR.join(lines) + '\n</tt>\n'
179
part.set_payload(payload)
180
# We're replacing the payload with the decoded payload so this
181
# will just get in the way.
182
del part['content-transfer-encoding']
183
omask = os.umask(002)
185
url = save_attachment(mlist, part, dir, filter_html=0)
188
part.set_payload(_("""\
189
An HTML attachment was scrubbed...
192
part.set_type('text/plain')
193
elif ctype == 'message/rfc822':
194
# This part contains a submessage, so it too needs scrubbing
195
submsg = part.get_payload(0)
196
omask = os.umask(002)
198
url = save_attachment(mlist, part, dir)
201
subject = submsg.get('subject', _('no subject'))
202
date = submsg.get('date', _('no date'))
203
who = submsg.get('from', _('unknown sender'))
204
size = len(str(submsg))
205
part.set_payload(_("""\
206
An embedded message was scrubbed...
213
part.set_type('text/plain')
214
# If the message isn't a multipart, then we'll strip it out as an
215
# attachment that would have to be separately downloaded. Pipermail
216
# will transform the url into a hyperlink.
217
elif not part.is_multipart():
218
payload = part.get_payload()
219
ctype = part.get_type()
221
omask = os.umask(002)
223
url = save_attachment(mlist, part, dir)
226
desc = part.get('content-description', _('not available'))
227
filename = part.get_filename(_('not available'))
228
part.set_payload(_("""\
229
A non-text attachment was scrubbed...
236
part.set_type('text/plain')
238
# We still have to sanitize multipart messages to flat text because
239
# Pipermail can't handle messages with list payloads. This is a kludge;
240
# def (n) clever hack ;).
241
if msg.is_multipart():
242
# By default we take the charset of the first text/plain part in the
243
# message, but if there was none, we'll use the list's preferred
244
# language's charset.
246
charset = Utils.GetCharSet(mlist.preferred_language)
247
# We now want to concatenate all the parts which have been scrubbed to
248
# text/plain, into a single text/plain payload. We need to make sure
249
# all the characters in the concatenated string are in the same
250
# encoding, so we'll use the 'replace' key in the coercion call.
251
# BAW: Martin's original patch suggested we might want to try
252
# generalizing to utf-8, and that's probably a good idea (eventually).
254
for part in msg.get_payload():
255
# All parts should be scrubbed to text/plain by now.
256
partctype = part.get_content_type()
257
if partctype <> 'text/plain':
258
text.append(_('Skipped content of type %(partctype)s'))
261
t = part.get_payload(decode=1)
262
except binascii.Error:
263
t = part.get_payload()
264
partcharset = part.get_charset()
265
if partcharset and partcharset <> charset:
267
t = unicode(t, partcharset, 'replace')
268
# Should use HTML-Escape, or try generalizing to UTF-8
269
t = t.encode(charset, 'replace')
271
# Replace funny characters
272
t = unicode(t, 'ascii', 'replace').encode('ascii')
274
# Now join the text and set the payload
275
sep = _('-------------- next part --------------\n')
276
msg.set_payload(sep.join(text), charset)
277
msg.set_type('text/plain')
278
del msg['content-transfer-encoding']
279
msg.add_header('Content-Transfer-Encoding', '8bit')
285
# Create all the directories to store this attachment in
287
os.makedirs(dir, 02775)
289
if e.errno <> errno.EEXIST: raise
290
# Unfortunately, FreeBSD seems to be broken in that it doesn't honor the
291
# mode arg of mkdir().
292
def twiddle(arg, dirname, names):
293
os.chmod(dirname, 02775)
294
os.path.walk(dir, twiddle, None)
298
def save_attachment(mlist, msg, dir, filter_html=1):
299
fsdir = os.path.join(mlist.archive_dir(), dir)
301
# Figure out the attachment type and get the decoded data
302
decodedpayload = msg.get_payload(decode=1)
303
# BAW: mimetypes ought to handle non-standard, but commonly found types,
304
# e.g. image/jpg (should be image/jpeg). For now we just store such
305
# things as application/octet-streams since that seems the safest.
306
ext = mimetypes.guess_extension(msg.get_type())
308
# We don't know what it is, so assume it's just a shapeless
309
# application/octet-stream, unless the Content-Type: is
310
# message/rfc822, in which case we know we'll coerce the type to
312
if msg.get_type() == 'message/rfc822':
317
# We need a lock to calculate the next attachment number
318
lockfile = os.path.join(fsdir, 'attachments.lock')
319
lock = LockFile.LockFile(lockfile)
322
# Now base the filename on what's in the attachment, uniquifying it if
324
filename = msg.get_filename()
326
filebase = 'attachment'
328
# Sanitize the filename given in the message headers
329
parts = pre.split(filename)
331
# Strip off leading dots
332
filename = dre.sub('', filename)
333
# Allow only alphanumerics, dash, underscore, and dot
334
filename = sre.sub('', filename)
335
# If the filename's extension doesn't match the type we guessed,
336
# which one should we go with? For now, let's go with the one we
337
# guessed so attachments can't lie about their type. Also, if the
338
# filename /has/ no extension, then tack on the one we guessed.
339
filebase, ignore = os.path.splitext(filename)
340
# Now we're looking for a unique name for this file on the file
341
# system. If msgdir/filebase.ext isn't unique, we'll add a counter
342
# after filebase, e.g. msgdir/filebase-cnt.ext
346
path = os.path.join(fsdir, filebase + extra + ext)
347
# Generally it is not a good idea to test for file existance
348
# before just trying to create it, but the alternatives aren't
349
# wonderful (i.e. os.open(..., O_CREAT | O_EXCL) isn't
350
# NFS-safe). Besides, we have an exclusive lock now, so we're
351
# guaranteed that no other process will be racing with us.
352
if os.path.exists(path):
354
extra = '-%04d' % counter
359
# `path' now contains the unique filename for the attachment. There's
360
# just one more step we need to do. If the part is text/html and
361
# ARCHIVE_HTML_SANITIZER is a string (which it must be or we wouldn't be
362
# here), then send the attachment through the filter program for
364
if filter_html and msg.get_type() == 'text/html':
365
base, ext = os.path.splitext(path)
366
tmppath = base + '-tmp' + ext
367
fp = open(tmppath, 'w')
369
fp.write(decodedpayload)
371
cmd = mm_cfg.ARCHIVE_HTML_SANITIZER % {'filename' : tmppath}
372
progfp = os.popen(cmd, 'r')
373
decodedpayload = progfp.read()
374
status = progfp.close()
377
'HTML sanitizer exited with non-zero status: %s',
381
# BAW: Since we've now sanitized the document, it should be plain
382
# text. Blarg, we really want the sanitizer to tell us what the type
383
# if the return data is. :(
386
# Is it a message/rfc822 attachment?
387
elif msg.get_type() == 'message/rfc822':
388
submsg = msg.get_payload()
389
# BAW: I'm sure we can eventually do better than this. :(
390
decodedpayload = Utils.websafe(str(submsg))
392
fp.write(decodedpayload)
394
# Now calculate the url
395
baseurl = mlist.GetBaseArchiveURL()
396
# Private archives will likely have a trailing slash. Normalize.
397
if baseurl[-1] <> '/':
399
url = baseurl + '%s/%s%s%s' % (dir, filebase, extra, ext)
added
removed
Mailman/Handlers/Scrubber.py
