~abompard/mailman/import21

« back to all changes in this revision

Viewing changes to src/mailman/utilities/importer.py

  • Committer: Aurélien Bompard
  • Date: 2013-10-08 08:55:24 UTC
  • Revision ID: aurelien@bompard.org-20131008085524-s3qe3whcwj9os2sr
Some more unicode checks when importing

Show diffs side-by-side

added added

removed removed

Lines of Context:
22
22
__metaclass__ = type
23
23
__all__ = [
24
24
    'import_config_pck',
25
 
    'ImportError',
26
25
    ]
27
26
 
28
27
 
32
31
from urllib2 import URLError
33
32
 
34
33
from mailman.config import config
35
 
from mailman.core.errors import MailmanError
36
34
from mailman.interfaces.action import FilterAction, Action
37
35
from mailman.interfaces.autorespond import ResponseAction
38
36
from mailman.interfaces.digests import DigestFrequency
44
42
from mailman.interfaces.bounce import UnrecognizedBounceDisposition
45
43
from mailman.interfaces.usermanager import IUserManager
46
44
from mailman.interfaces.member import DeliveryMode, DeliveryStatus, MemberRole
47
 
from mailman.interfaces.languages import ILanguageManager
48
45
from mailman.handlers.decorate import decorate, decorate_template
49
46
from mailman.utilities.i18n import search
50
47
from zope.component import getUtility
51
48
 
52
49
 
53
50
 
54
 
class Import21Error(MailmanError):
55
 
    pass
56
 
 
57
 
 
58
 
def str_to_unicode(value):
59
 
    # Convert a string to unicode when the encoding is not declared
60
 
    if isinstance(value, unicode):
61
 
        return value
62
 
    for encoding in ("ascii", "utf-8"):
63
 
        try:
64
 
            return unicode(value, encoding)
65
 
        except UnicodeDecodeError, e:
66
 
            continue
67
 
    # we did our best, use replace
68
 
    return unicode(value, 'ascii', 'replace')
69
 
 
70
 
 
71
51
def seconds_to_delta(value):
72
52
    return datetime.timedelta(seconds=value)
73
53
 
127
107
def unicode_to_string(value):
128
108
    return str(value) if value is not None else None
129
109
 
130
 
 
131
 
def check_language_code(code):
132
 
    if code is None:
133
 
        return None
134
 
    code = unicode(code)
135
 
    if code not in getUtility(ILanguageManager):
136
 
        msg = """Missing language: {0}
137
 
You must add a section describing this language in your mailman.cfg file.
138
 
This section should look like this:
139
 
[language.{0}]
140
 
# The English name for the language.
141
 
description: CHANGE ME
142
 
# And the default character set for the language.
143
 
charset: utf-8
144
 
# Whether the language is enabled or not.
145
 
enabled: yes
146
 
""".format(code)
147
 
        raise Import21Error(msg)
148
 
    return code
149
 
 
150
110
 
151
111
# Attributes in Mailman 2 which have a different type in Mailman 3.
152
112
TYPES = dict(
169
129
    default_member_action=member_action_mapping,
170
130
    default_nonmember_action=nonmember_action_mapping,
171
131
    moderator_password=unicode_to_string,
172
 
    preferred_language=check_language_code,
173
132
    )
174
133
 
175
134
 
176
135
# Attribute names in Mailman 2 which are renamed in Mailman 3.
177
136
NAME_MAPPINGS = dict(
 
137
    host_name='mail_host',
178
138
    include_list_post_header='allow_list_posts',
179
139
    real_name='display_name',
180
140
    last_post_time='last_post_at',
220
180
        # Handle the simple case where the key is an attribute of the
221
181
        # IMailingList and the types are the same (modulo 8-bit/unicode
222
182
        # strings).
223
 
        # When attributes raise an exception, hasattr may think they don't
224
 
        # exist (see python issue 9666). Add them here.
225
 
        if hasattr(mlist, key) or key in ("preferred_language", ):
 
183
        if hasattr(mlist, key):
226
184
            if isinstance(value, str):
227
 
                value = str_to_unicode(value)
 
185
                for encoding in ("ascii", "utf-8"):
 
186
                    try:
 
187
                        value = unicode(value, encoding)
 
188
                    except UnicodeDecodeError, e:
 
189
                        continue
 
190
                    else:
 
191
                        break
 
192
                if isinstance(value, str): # we did our best
 
193
                    value = unicode(value, 'ascii', 'replace')
228
194
            # Some types require conversion.
229
195
            converter = TYPES.get(key)
230
196
            if converter is not None:
244
210
        mlist.archive_policy = ArchivePolicy.never
245
211
    # Handle ban list
246
212
    for addr in config_dict.get('ban_list', []):
247
 
        IBanManager(mlist).ban(str_to_unicode(addr))
 
213
        IBanManager(mlist).ban(unicode(addr))
248
214
    # Handle acceptable aliases
249
 
    acceptable_aliases = config_dict.get('acceptable_aliases', '')
250
 
    if isinstance(acceptable_aliases, basestring):
251
 
        acceptable_aliases = acceptable_aliases.splitlines()
252
 
    for addr in acceptable_aliases:
 
215
    for addr in config_dict.get('acceptable_aliases', '').splitlines():
253
216
        addr = addr.strip()
254
217
        if not addr:
255
218
            continue
256
 
        addr = str_to_unicode(addr)
257
 
        try:
258
 
            IAcceptableAliasSet(mlist).add(addr)
259
 
        except ValueError:
260
 
            IAcceptableAliasSet(mlist).add("^" + addr)
 
219
        IAcceptableAliasSet(mlist).add(unicode(addr))
261
220
    # Handle conversion to URIs
262
221
    convert_to_uri = {
263
222
        "welcome_msg": "welcome_message_uri",
353
312
    """
354
313
    usermanager = getUtility(IUserManager)
355
314
    for email in members:
356
 
        # for owners and members, the emails can have a mixed case, so
357
 
        # lowercase them all
358
 
        email = str_to_unicode(email).lower()
 
315
        email = unicode(email)
359
316
        roster = mlist.get_roster(role)
360
317
        if roster.get_member(email) is not None:
361
318
            print("%s is already imported with role %s" % (email, role),
362
319
                  file=sys.stderr)
363
320
            continue
364
 
        address = usermanager.get_address(email)
365
321
        user = usermanager.get_user(email)
366
322
        if user is None:
367
 
            user = usermanager.create_user()
368
 
            if address is None:
369
 
                merged_members = {}
370
 
                merged_members.update(config_dict.get("members", {}))
371
 
                merged_members.update(config_dict.get("digest_members", {}))
372
 
                if merged_members.get(email, 0) != 0:
373
 
                    original_email = str_to_unicode(merged_members[email])
374
 
                else:
375
 
                    original_email = email
376
 
                address = usermanager.create_address(original_email)
377
 
                address.verified_on = datetime.datetime.now()
378
 
            user.link(address)
 
323
            merged_members = {}
 
324
            merged_members.update(config_dict.get("members", {}))
 
325
            merged_members.update(config_dict.get("digest_members", {}))
 
326
            if merged_members.get(email, 0) != 0:
 
327
                original_email = merged_members[email]
 
328
            else:
 
329
                original_email = email
 
330
            user = usermanager.create_user(unicode(original_email))
 
331
        address = usermanager.get_address(email)
 
332
        address.verified_on = datetime.datetime.now()
379
333
        mlist.subscribe(address, role)
380
334
        member = roster.get_member(email)
381
335
        assert member is not None
392
346
            pass
393
347
        if email in config_dict.get("language", {}):
394
348
            member.preferences.preferred_language = \
395
 
                check_language_code(config_dict["language"][email])
 
349
                unicode(config_dict["language"][email])
396
350
        # if the user already exists, display_name and password will be
397
351
        # overwritten
398
352
        if email in config_dict.get("usernames", {}):
399
 
            address.display_name = \
400
 
                    str_to_unicode(config_dict["usernames"][email])
401
 
            user.display_name    = \
402
 
                    str_to_unicode(config_dict["usernames"][email])
 
353
            address.display_name = unicode(config_dict["usernames"][email])
 
354
            user.display_name    = unicode(config_dict["usernames"][email])
403
355
        if email in config_dict.get("passwords", {}):
404
356
            user.password = config.password_context.encrypt(
405
357
                                    config_dict["passwords"][email])