1
# -*- test-case-name: twisted.mail.test.test_pop3client -*-
2
# Copyright (c) 2001-2004 Divmod Inc.
3
# Copyright (c) 2008 Twisted Matrix Laboratories.
4
# See LICENSE for details.
7
POP3 client protocol implementation
9
Don't use this module directly. Use twisted.mail.pop3 instead.
16
from twisted.python import log
17
from twisted.python.hashlib import md5
18
from twisted.internet import defer
19
from twisted.protocols import basic
20
from twisted.protocols import policies
21
from twisted.internet import error
22
from twisted.internet import interfaces
27
class POP3ClientError(Exception):
28
"""Base class for all exceptions raised by POP3Client.
31
class InsecureAuthenticationDisallowed(POP3ClientError):
32
"""Secure authentication was required but no mechanism could be found.
35
class TLSError(POP3ClientError):
37
Secure authentication was required but either the transport does
38
not support TLS or no TLS context factory was supplied.
41
class TLSNotSupportedError(POP3ClientError):
43
Secure authentication was required but the server does not support
47
class ServerErrorResponse(POP3ClientError):
48
"""The server returned an error response to a request.
50
def __init__(self, reason, consumer=None):
51
POP3ClientError.__init__(self, reason)
52
self.consumer = consumer
54
class LineTooLong(POP3ClientError):
55
"""The server sent an extremely long line.
59
# Internal helper. POP3 responses sometimes occur in the
60
# form of a list of lines containing two pieces of data,
61
# a message index and a value of some sort. When a message
62
# is deleted, it is omitted from these responses. The
63
# setitem method of this class is meant to be called with
64
# these two values. In the cases where indexes are skipped,
65
# it takes care of padding out the missing values with None.
66
def __init__(self, L):
68
def setitem(self, (item, value)):
69
diff = item - len(self.L) + 1
71
self.L.extend([None] * diff)
76
# Parse a STAT response
77
numMsgs, totalSize = line.split(None, 1)
78
return int(numMsgs), int(totalSize)
82
# Parse a LIST response
83
index, size = line.split(None, 1)
84
return int(index) - 1, int(size)
88
# Parse a UIDL response
89
index, uid = line.split(None, 1)
90
return int(index) - 1, uid
92
def _codeStatusSplit(line):
93
# Parse an +OK or -ERR response
94
parts = line.split(' ', 1)
99
def _dotUnquoter(line):
101
C{'.'} characters which begin a line of a message are doubled to avoid
102
confusing with the terminating C{'.\\r\\n'} sequence. This function
105
if line.startswith('..'):
109
class POP3Client(basic.LineOnlyReceiver, policies.TimeoutMixin):
110
"""POP3 client protocol implementation class
112
Instances of this class provide a convenient, efficient API for
113
retrieving and deleting messages from a POP3 server.
115
@type startedTLS: C{bool}
116
@ivar startedTLS: Whether TLS has been negotiated successfully.
119
@type allowInsecureLogin: C{bool}
120
@ivar allowInsecureLogin: Indicate whether login() should be
121
allowed if the server offers no authentication challenge and if
122
our transport does not offer any protection via encryption.
124
@type serverChallenge: C{str} or C{None}
125
@ivar serverChallenge: Challenge received from the server
127
@type timeout: C{int}
128
@ivar timeout: Number of seconds to wait before timing out a
129
connection. If the number is <= 0, no timeout checking will be
134
allowInsecureLogin = False
136
serverChallenge = None
138
# Capabilities are not allowed to change during the session
139
# (except when TLS is negotiated), so cache the first response and
140
# use that for all later lookups
143
# Regular expression to search for in the challenge string in the server
145
_challengeMagicRe = re.compile('(<[^>]+>)')
147
# List of pending calls.
148
# We are a pipelining API but don't actually
149
# support pipelining on the network yet.
152
# The Deferred to which the very next result will go.
155
# Whether we dropped the connection because of a timeout
158
# If the server sends an initial -ERR, this is the message it sent
160
_greetingError = None
162
def _blocked(self, f, *a):
163
# Internal helper. If commands are being blocked, append
164
# the given command and arguments to a list and return a Deferred
165
# that will be chained with the return value of the function
166
# when it eventually runs. Otherwise, set up for commands to be
168
# blocked and return None.
169
if self._blockedQueue is not None:
171
self._blockedQueue.append((d, f, a))
173
self._blockedQueue = []
177
# Internal helper. Indicate that a function has completed.
178
# If there are blocked commands, run the next one. If there
179
# are not, set up for the next command to not be blocked.
180
if self._blockedQueue == []:
181
self._blockedQueue = None
182
elif self._blockedQueue is not None:
183
_blockedQueue = self._blockedQueue
184
self._blockedQueue = None
186
d, f, a = _blockedQueue.pop(0)
189
# f is a function which uses _blocked (otherwise it wouldn't
190
# have gotten into the blocked queue), which means it will have
191
# re-set _blockedQueue to an empty list, so we can put the rest
192
# of the blocked queue back into it now.
193
self._blockedQueue.extend(_blockedQueue)
196
def sendShort(self, cmd, args):
197
# Internal helper. Send a command to which a short response
198
# is expected. Return a Deferred that fires when the response
199
# is received. Block all further commands from being sent until
200
# the response is received. Transition the state to SHORT.
201
d = self._blocked(self.sendShort, cmd, args)
206
self.sendLine(cmd + ' ' + args)
210
self._waiting = defer.Deferred()
213
def sendLong(self, cmd, args, consumer, xform):
214
# Internal helper. Send a command to which a multiline
215
# response is expected. Return a Deferred that fires when
216
# the entire response is received. Block all further commands
217
# from being sent until the entire response is received.
218
# Transition the state to LONG_INITIAL.
219
d = self._blocked(self.sendLong, cmd, args, consumer, xform)
224
self.sendLine(cmd + ' ' + args)
227
self.state = 'LONG_INITIAL'
229
self._consumer = consumer
230
self._waiting = defer.Deferred()
233
# Twisted protocol callback
234
def connectionMade(self):
236
self.setTimeout(self.timeout)
238
self.state = 'WELCOME'
239
self._blockedQueue = []
241
def timeoutConnection(self):
242
self._timedOut = True
243
self.transport.loseConnection()
245
def connectionLost(self, reason):
247
self.setTimeout(None)
250
reason = error.TimeoutError()
251
elif self._greetingError:
252
reason = ServerErrorResponse(self._greetingError)
255
if self._waiting is not None:
256
d.append(self._waiting)
258
if self._blockedQueue is not None:
259
d.extend([deferred for (deferred, f, a) in self._blockedQueue])
260
self._blockedQueue = None
264
def lineReceived(self, line):
270
state = getattr(self, 'state_' + state)(line) or state
271
if self.state is None:
274
def lineLengthExceeded(self, buffer):
275
# XXX - We need to be smarter about this
276
if self._waiting is not None:
277
waiting, self._waiting = self._waiting, None
278
waiting.errback(LineTooLong())
279
self.transport.loseConnection()
281
# POP3 Client state logic - don't touch this.
282
def state_WELCOME(self, line):
283
# WELCOME is the first state. The server sends one line of text
284
# greeting us, possibly with an APOP challenge. Transition the
286
code, status = _codeStatusSplit(line)
288
self._greetingError = status
289
self.transport.loseConnection()
291
m = self._challengeMagicRe.search(status)
294
self.serverChallenge = m.group(1)
296
self.serverGreeting(status)
301
def state_WAITING(self, line):
302
# The server isn't supposed to send us anything in this state.
303
log.msg("Illegal line from server: " + repr(line))
305
def state_SHORT(self, line):
306
# This is the state we are in when waiting for a single
307
# line response. Parse it and fire the appropriate callback
308
# or errback. Transition the state back to WAITING.
309
deferred, self._waiting = self._waiting, None
311
code, status = _codeStatusSplit(line)
313
deferred.callback(status)
315
deferred.errback(ServerErrorResponse(status))
318
def state_LONG_INITIAL(self, line):
319
# This is the state we are in when waiting for the first
320
# line of a long response. Parse it and transition the
321
# state to LONG if it is an okay response; if it is an
322
# error response, fire an errback, clean up the things
323
# waiting for a long response, and transition the state
325
code, status = _codeStatusSplit(line)
328
consumer = self._consumer
329
deferred = self._waiting
330
self._consumer = self._waiting = self._xform = None
332
deferred.errback(ServerErrorResponse(status, consumer))
335
def state_LONG(self, line):
336
# This is the state for each line of a long response.
337
# If it is the last line, finish things, fire the
338
# Deferred, and transition the state to WAITING.
339
# Otherwise, pass the line to the consumer.
341
consumer = self._consumer
342
deferred = self._waiting
343
self._consumer = self._waiting = self._xform = None
345
deferred.callback(consumer)
348
if self._xform is not None:
349
self._consumer(self._xform(line))
355
# Callbacks - override these
356
def serverGreeting(self, greeting):
357
"""Called when the server has sent us a greeting.
359
@type greeting: C{str} or C{None}
360
@param greeting: The status message sent with the server
361
greeting. For servers implementing APOP authentication, this
362
will be a challenge string. .
366
# External API - call these (most of 'em anyway)
367
def startTLS(self, contextFactory=None):
369
Initiates a 'STLS' request and negotiates the TLS / SSL
372
@type contextFactory: C{ssl.ClientContextFactory} @param
373
contextFactory: The context factory with which to negotiate
374
TLS. If C{None}, try to create a new one.
376
@return: A Deferred which fires when the transport has been
377
secured according to the given contextFactory, or which fails
378
if the transport cannot be secured.
380
tls = interfaces.ITLSTransport(self.transport, None)
382
return defer.fail(TLSError(
383
"POP3Client transport does not implement "
384
"interfaces.ITLSTransport"))
386
if contextFactory is None:
387
contextFactory = self._getContextFactory()
389
if contextFactory is None:
390
return defer.fail(TLSError(
391
"POP3Client requires a TLS context to "
392
"initiate the STLS handshake"))
394
d = self.capabilities()
395
d.addCallback(self._startTLS, contextFactory, tls)
399
def _startTLS(self, caps, contextFactory, tls):
400
assert not self.startedTLS, "Client and Server are currently communicating via TLS"
402
if 'STLS' not in caps:
403
return defer.fail(TLSNotSupportedError(
404
"Server does not support secure communication "
407
d = self.sendShort('STLS', None)
408
d.addCallback(self._startedTLS, contextFactory, tls)
409
d.addCallback(lambda _: self.capabilities())
413
def _startedTLS(self, result, context, tls):
415
self.transport.startTLS(context)
416
self._capCache = None
417
self.startedTLS = True
421
def _getContextFactory(self):
423
from twisted.internet import ssl
427
context = ssl.ClientContextFactory()
428
context.method = ssl.SSL.TLSv1_METHOD
432
def login(self, username, password):
433
"""Log into the server.
435
If APOP is available it will be used. Otherwise, if TLS is
436
available an 'STLS' session will be started and plaintext
437
login will proceed. Otherwise, if the instance attribute
438
allowInsecureLogin is set to True, insecure plaintext login
439
will proceed. Otherwise, InsecureAuthenticationDisallowed
440
will be raised (asynchronously).
442
@param username: The username with which to log in.
443
@param password: The password with which to log in.
446
@return: A deferred which fires when login has
449
d = self.capabilities()
450
d.addCallback(self._login, username, password)
454
def _login(self, caps, username, password):
455
if self.serverChallenge is not None:
456
return self._apop(username, password, self.serverChallenge)
458
tryTLS = 'STLS' in caps
460
#If our transport supports switching to TLS, we might want to try to switch to TLS.
461
tlsableTransport = interfaces.ITLSTransport(self.transport, None) is not None
463
# If our transport is not already using TLS, we might want to try to switch to TLS.
464
nontlsTransport = interfaces.ISSLTransport(self.transport, None) is None
466
if not self.startedTLS and tryTLS and tlsableTransport and nontlsTransport:
469
d.addCallback(self._loginTLS, username, password)
472
elif self.startedTLS or not nontlsTransport or self.allowInsecureLogin:
473
return self._plaintext(username, password)
475
return defer.fail(InsecureAuthenticationDisallowed())
478
def _loginTLS(self, res, username, password):
479
return self._plaintext(username, password)
481
def _plaintext(self, username, password):
482
# Internal helper. Send a username/password pair, returning a Deferred
483
# that fires when both have succeeded or fails when the server rejects
485
return self.user(username).addCallback(lambda r: self.password(password))
487
def _apop(self, username, password, challenge):
488
# Internal helper. Computes and sends an APOP response. Returns
489
# a Deferred that fires when the server responds to the response.
490
digest = md5(challenge + password).hexdigest()
491
return self.apop(username, digest)
493
def apop(self, username, digest):
494
"""Perform APOP login.
496
This should be used in special circumstances only, when it is
497
known that the server supports APOP authentication, and APOP
498
authentication is absolutely required. For the common case,
499
use L{login} instead.
501
@param username: The username with which to log in.
502
@param digest: The challenge response to authenticate with.
504
return self.sendShort('APOP', username + ' ' + digest)
506
def user(self, username):
507
"""Send the user command.
509
This performs the first half of plaintext login. Unless this
510
is absolutely required, use the L{login} method instead.
512
@param username: The username with which to log in.
514
return self.sendShort('USER', username)
516
def password(self, password):
517
"""Send the password command.
519
This performs the second half of plaintext login. Unless this
520
is absolutely required, use the L{login} method instead.
522
@param password: The plaintext password with which to authenticate.
524
return self.sendShort('PASS', password)
526
def delete(self, index):
527
"""Delete a message from the server.
530
@param index: The index of the message to delete.
534
@return: A deferred which fires when the delete command
535
is successful, or fails if the server returns an error.
537
return self.sendShort('DELE', str(index + 1))
539
def _consumeOrSetItem(self, cmd, args, consumer, xform):
540
# Internal helper. Send a long command. If no consumer is
541
# provided, create a consumer that puts results into a list
542
# and return a Deferred that fires with that list when it
546
consumer = _ListSetter(L).setitem
547
return self.sendLong(cmd, args, consumer, xform).addCallback(lambda r: L)
548
return self.sendLong(cmd, args, consumer, xform)
550
def _consumeOrAppend(self, cmd, args, consumer, xform):
551
# Internal helper. Send a long command. If no consumer is
552
# provided, create a consumer that appends results to a list
553
# and return a Deferred that fires with that list when it is
558
return self.sendLong(cmd, args, consumer, xform).addCallback(lambda r: L)
559
return self.sendLong(cmd, args, consumer, xform)
561
def capabilities(self, useCache=True):
562
"""Retrieve the capabilities supported by this server.
564
Not all servers support this command. If the server does not
565
support this, it is treated as though it returned a successful
566
response listing no capabilities. At some future time, this may be
567
changed to instead seek out information about a server's
568
capabilities in some other fashion (only if it proves useful to do
569
so, and only if there are servers still in use which do not support
570
CAPA but which do support POP3 extensions that are useful).
572
@type useCache: C{bool}
573
@param useCache: If set, and if capabilities have been
574
retrieved previously, just return the previously retrieved
577
@return: A Deferred which fires with a C{dict} mapping C{str}
578
to C{None} or C{list}s of C{str}. For example::
581
S: +OK Capability list follows
584
S: SASL CRAM-MD5 KERBEROS_V4
590
S: IMPLEMENTATION Shlemazle-Plotz-v302
593
will be lead to a result of::
597
| 'SASL': ['CRAM-MD5', 'KERBEROS_V4'],
598
| 'RESP-CODES': None,
599
| 'LOGIN-DELAY': ['900'],
600
| 'PIPELINING': None,
603
| 'IMPLEMENTATION': ['Shlemazle-Plotz-v302']}
605
if useCache and self._capCache is not None:
606
return defer.succeed(self._capCache)
614
cache[tmp[0]] = tmp[1:]
616
def capaNotSupported(err):
617
err.trap(ServerErrorResponse)
620
def gotCapabilities(result):
621
self._capCache = cache
624
d = self._consumeOrAppend('CAPA', None, consume, None)
625
d.addErrback(capaNotSupported).addCallback(gotCapabilities)
630
"""Do nothing, with the help of the server.
632
No operation is performed. The returned Deferred fires when
635
return self.sendShort("NOOP", None)
639
"""Remove the deleted flag from any messages which have it.
641
The returned Deferred fires when the server responds.
643
return self.sendShort("RSET", None)
646
def retrieve(self, index, consumer=None, lines=None):
647
"""Retrieve a message from the server.
649
If L{consumer} is not None, it will be called with
650
each line of the message as it is received. Otherwise,
651
the returned Deferred will be fired with a list of all
652
the lines when the message has been completely received.
656
return self._consumeOrAppend('RETR', idx, consumer, _dotUnquoter)
658
return self._consumeOrAppend('TOP', '%s %d' % (idx, lines), consumer, _dotUnquoter)
662
"""Get information about the size of this mailbox.
664
The returned Deferred will be fired with a tuple containing
665
the number or messages in the mailbox and the size (in bytes)
668
return self.sendShort('STAT', None).addCallback(_statXform)
671
def listSize(self, consumer=None):
672
"""Retrieve a list of the size of all messages on the server.
674
If L{consumer} is not None, it will be called with two-tuples
675
of message index number and message size as they are received.
676
Otherwise, a Deferred which will fire with a list of B{only}
677
message sizes will be returned. For messages which have been
678
deleted, None will be used in place of the message size.
680
return self._consumeOrSetItem('LIST', None, consumer, _listXform)
683
def listUID(self, consumer=None):
684
"""Retrieve a list of the UIDs of all messages on the server.
686
If L{consumer} is not None, it will be called with two-tuples
687
of message index number and message UID as they are received.
688
Otherwise, a Deferred which will fire with of list of B{only}
689
message UIDs will be returned. For messages which have been
690
deleted, None will be used in place of the message UID.
692
return self._consumeOrSetItem('UIDL', None, consumer, _uidXform)
696
"""Disconnect from the server.
698
return self.sendShort('QUIT', None)
702
'InsecureAuthenticationDisallowed', 'LineTooLong', 'POP3ClientError',
703
'ServerErrorResponse', 'TLSError', 'TLSNotSupportedError',