← Back to branch summary

~sambuddhabasu1/mailman/fix_mailman_run_error

« back to all changes in this revision

Viewing changes to src/mailman/runners/archive.py

  • Committer: Barry Warsaw
  • Date: 2012-03-26 12:04:00 UTC
  • Revision ID: barry@list.org-20120326120400-jfezy6cg60ygod7k
Architecture
------------
 * Internally, all datetimes are kept in the UTC timezone, however because of
   LP: #280708, they are stored in the database in naive format.
 * `received_time` is now added to the message metadata by the LMTP runner
   instead of by `Switchboard.enqueue()`.  This latter no longer depends on
   `received_time` in the metadata.
 * The `ArchiveRunner` no longer acquires a lock before it calls the
   individual archiver implementations, since not all of them need a lock.  If
   they do, the implementations must acquire said lock themselves.

Configuration
-------------
 * New configuration variables `clobber_date` and `clobber_skew` supported in
   every `[archiver.<name>]` section.  These are used to determine under what
   circumstances a message destined for a specific archiver should have its
   `Date:` header clobbered.

Show diffs side-by-side

added added

removed removed

Lines of Context:
src/mailman/runners/archive.py
25
25
    ]
26
26
 
27
27
 
28
 
import os
 
28
import copy
29
29
import logging
30
30
 
 
31
from email.utils import parsedate_tz, mktime_tz
31
32
from datetime import datetime
32
 
from email.utils import parsedate_tz, mktime_tz, formatdate
33
 
from flufl.lock import Lock
34
33
from lazr.config import as_timedelta
35
34
 
36
35
from mailman.config import config
37
36
from mailman.core.runner import Runner
 
37
from mailman.interfaces.archiver import ClobberDate
 
38
from mailman.utilities.datetime import RFC822_DATE_FMT, now
 
39
 
38
40
 
39
41
log = logging.getLogger('mailman.error')
40
42
 
41
43
 
42
44
 
 
45
def _should_clobber(msg, msgdata, archiver):
 
46
    """Should the Date header in the original message get clobbered?"""
 
47
    # Calculate the Date header of the message as a datetime.  What if there
 
48
    # are multiple Date headers, even in violation of the RFC?  For now, take
 
49
    # the first one.  If there are no Date headers, then definitely clobber.
 
50
    original_date = msg.get('date')
 
51
    if original_date is None:
 
52
        return True
 
53
    section = getattr(config.archiver, archiver, None)
 
54
    if section is None:
 
55
        log.error('No archiver config section found: {0}'.format(archiver))
 
56
        return False
 
57
    try:
 
58
        clobber = ClobberDate[section.clobber_date]
 
59
    except ValueError:
 
60
        log.error('Invalid clobber_date for "{0}": {1}'.format(
 
61
            archiver, section.clobber_date))
 
62
        return False
 
63
    if clobber is ClobberDate.always:
 
64
        return True
 
65
    elif clobber is ClobberDate.never:
 
66
        return False
 
67
    # Maybe we'll clobber the date.  Let's see if it's farther off from now
 
68
    # than the skew period.
 
69
    skew = as_timedelta(section.clobber_skew)
 
70
    try:
 
71
        time_tuple = parsedate_tz(original_date)
 
72
    except (ValueError, OverflowError):
 
73
        # The likely cause of this is that the year in the Date: field is
 
74
        # horribly incorrect, e.g. (from SF bug # 571634):
 
75
        #
 
76
        # Date: Tue, 18 Jun 0102 05:12:09 +0500
 
77
        #
 
78
        # Obviously clobber such dates.
 
79
        return True
 
80
    if time_tuple is None:
 
81
        # There was some other bogosity in the Date header.
 
82
        return True
 
83
    claimed_date = datetime.fromtimestamp(mktime_tz(time_tuple))
 
84
    return (abs(now() - claimed_date) > skew)
 
85
 
 
86
 
 
87
 
43
88
class ArchiveRunner(Runner):
44
89
    """The archive runner."""
45
90
 
46
91
    def _dispose(self, mlist, msg, msgdata):
47
 
        # Support clobber_date, i.e. setting the date in the archive to the
48
 
        # received date, not the (potentially bogus) Date: header of the
49
 
        # original message.
50
 
        clobber = False
51
 
        original_date = msg.get('date')
52
 
        received_time = formatdate(msgdata['received_time'])
53
 
        # FIXME 2012-03-23 BAW: LP: #963612
54
 
        ## if not original_date:
55
 
        ##     clobber = True
56
 
        ## elif int(config.archiver.pipermail.clobber_date_policy) == 1:
57
 
        ##     clobber = True
58
 
        ## elif int(config.archiver.pipermail.clobber_date_policy) == 2:
59
 
        ##     # What's the timestamp on the original message?
60
 
        ##     timetup = parsedate_tz(original_date)
61
 
        ##     now = datetime.now()
62
 
        ##     try:
63
 
        ##         if not timetup:
64
 
        ##             clobber = True
65
 
        ##         else:
66
 
        ##             utc_timestamp = datetime.fromtimestamp(mktime_tz(timetup))
67
 
        ##             date_skew = as_timedelta(
68
 
        ##                 config.archiver.pipermail.allowable_sane_date_skew)
69
 
        ##             clobber = (abs(now - utc_timestamp) > date_skew)
70
 
        ##     except (ValueError, OverflowError):
71
 
        ##         # The likely cause of this is that the year in the Date: field
72
 
        ##         # is horribly incorrect, e.g. (from SF bug # 571634):
73
 
        ##         # Date: Tue, 18 Jun 0102 05:12:09 +0500
74
 
        ##         # Obviously clobber such dates.
75
 
        ##         clobber = True
76
 
        ## if clobber:
77
 
        ##     del msg['date']
78
 
        ##     del msg['x-original-date']
79
 
        ##     msg['Date'] = received_time
80
 
        ##     if original_date:
81
 
        ##         msg['X-Original-Date'] = original_date
82
 
        # Always put an indication of when we received the message.
83
 
        msg['X-List-Received-Date'] = received_time
84
 
        # While a list archiving lock is acquired, archive the message.
85
 
        with Lock(os.path.join(mlist.data_path, 'archive.lck')):
86
 
            for archiver in config.archivers:
87
 
                # A problem in one archiver should not prevent other archivers
88
 
                # from running.
89
 
                try:
90
 
                    archiver.archive_message(mlist, msg)
91
 
                except Exception:
92
 
                    log.exception('Broken archiver: %s' % archiver.name)
 
92
        received_time = msgdata.get('received_time', now(strip_tzinfo=False))
 
93
        for archiver in config.archivers:
 
94
            msg_copy = copy.deepcopy(msg)
 
95
            if _should_clobber(msg, msgdata, archiver.name):
 
96
                original_date = msg_copy['date']
 
97
                del msg_copy['date']
 
98
                del msg_copy['x-original-date']
 
99
                msg_copy['Date'] = received_time.strftime(RFC822_DATE_FMT)
 
100
                if original_date:
 
101
                    msg_copy['X-Original-Date'] = original_date
 
102
            # A problem in one archiver should not prevent other archivers
 
103
            # from running.
 
104
            try:
 
105
                archiver.archive_message(mlist, msg_copy)
 
106
            except Exception:
 
107
                log.exception('Broken archiver: %s' % archiver.name)