1
# -*- test-case-name: twisted.conch.test.test_cftp -*-
3
# Copyright (c) 2001-2004 Twisted Matrix Laboratories.
4
# See LICENSE for details.
7
# $Id: cftp.py,v 1.65 2004/03/11 00:29:14 z3p Exp $
9
#""" Implementation module for the `cftp` command.
11
from twisted.conch.client import agent, connect, default, options
12
from twisted.conch.error import ConchError
13
from twisted.conch.ssh import connection, common
14
from twisted.conch.ssh import channel, filetransfer
15
from twisted.protocols import basic
16
from twisted.internet import reactor, stdio, defer, utils
17
from twisted.python import log, usage, failure
19
import os, sys, getpass, struct, tty, fcntl, base64, signal, stat, errno
20
import fnmatch, pwd, time, glob
22
class ClientOptions(options.ConchOptions):
24
synopsis = """Usage: cftp [options] [user@]host
25
cftp [options] [user@]host[:dir[/]]
26
cftp [options] [user@]host[:file [localfile]]
30
['buffersize', 'B', 32768, 'Size of the buffer to use for sending/receiving.'],
31
['batchfile', 'b', None, 'File to read commands from, or \'-\' for stdin.'],
32
['requests', 'R', 5, 'Number of requests to make before waiting for a reply.'],
33
['subsystem', 's', 'sftp', 'Subsystem/server program to connect to.']]
34
zsh_altArgDescr = {"buffersize":"Size of send/receive buffer (default: 32768)"}
35
#zsh_multiUse = ["foo", "bar"]
36
#zsh_mutuallyExclusive = [("foo", "bar"), ("bar", "baz")]
37
#zsh_actions = {"foo":'_files -g "*.foo"', "bar":"(one two three)"}
38
#zsh_actionDescr = {"logfile":"log file name", "random":"random seed"}
39
zsh_extras = ['2::localfile:{if [[ $words[1] == *:* ]]; then; _files; fi}']
41
def parseArgs(self, host, localPath=None):
42
self['remotePath'] = ''
44
host, self['remotePath'] = host.split(':', 1)
45
self['remotePath'].rstrip('/')
47
self['localPath'] = localPath
51
# prof = hotshot.Profile('cftp.prof')
54
if '-l' in args: # cvs is an idiot
56
args = args[i:i+2]+args
58
options = ClientOptions()
60
options.parseOptions(args)
61
except usage.UsageError, u:
66
log.startLogging(sys.stderr)
76
from twisted.python import failure
82
log.err(failure.Failure())
85
def doConnect(options):
86
# log.deferr = handleError # HACK
87
if '@' in options['host']:
88
options['user'], options['host'] = options['host'].split('@',1)
89
host = options['host']
90
if not options['user']:
91
options['user'] = getpass.getuser()
92
if not options['port']:
95
options['port'] = int(options['port'])
96
host = options['host']
97
port = options['port']
98
conn = SSHConnection()
99
conn.options = options
100
vhk = default.verifyHostKey
101
uao = default.SSHUserAuthClient(options['user'], options, conn)
102
connect.connect(host, port, options, vhk, uao).addErrback(_ebExit)
106
if hasattr(f.value, 'value'):
111
#exitStatus = "conch: exiting with error %s" % f
116
def _ignore(*args): pass
120
def __init__(self, f):
123
f.seek(0, 2) # seek to the end
126
def __getattr__(self, attr):
127
return getattr(self.f, attr)
129
class StdioClient(basic.LineReceiver):
134
def __init__(self, client, f = None):
136
self.currentDirectory = ''
138
self.useProgressBar = (not f and 1) or 0
140
def connectionMade(self):
141
self.client.realPath('').addCallback(self._cbSetCurDir)
143
def _cbSetCurDir(self, path):
144
self.currentDirectory = path
147
def lineReceived(self, line):
148
if self.client.transport.localClosed:
150
log.msg('got line %s' % repr(line))
155
if self.file and line.startswith('-'):
156
self.ignoreErrors = 1
159
self.ignoreErrors = 0
161
command, rest = line.split(' ', 1)
164
command, rest = line, ''
165
if command.startswith('!'): # command
167
rest = (command[1:] + ' ' + rest).strip()
169
command = command.upper()
170
log.msg('looking up cmd %s' % command)
171
f = getattr(self, 'cmd_%s' % command, None)
173
d = defer.maybeDeferred(f, rest)
174
d.addCallback(self._cbCommand)
175
d.addErrback(self._ebCommand)
177
self._ebCommand(failure.Failure(NotImplementedError(
178
"No command called `%s'" % command)))
181
def _printFailure(self, f):
183
e = f.trap(NotImplementedError, filetransfer.SFTPError, OSError, IOError)
184
if e == NotImplementedError:
185
self.transport.write(self.cmd_HELP(''))
186
elif e == filetransfer.SFTPError:
187
self.transport.write("remote error %i: %s\n" %
188
(f.value.code, f.value.message))
189
elif e in (OSError, IOError):
190
self.transport.write("local error %i: %s\n" %
191
(f.value.errno, f.value.strerror))
194
if self.client.transport.localClosed:
196
self.transport.write(self.ps)
197
self.ignoreErrors = 0
199
l = self.file.readline()
201
self.client.transport.loseConnection()
203
self.transport.write(l)
204
self.lineReceived(l.strip())
206
def _cbCommand(self, result):
207
if result is not None:
208
self.transport.write(result)
209
if not result.endswith('\n'):
210
self.transport.write('\n')
213
def _ebCommand(self, f):
214
self._printFailure(f)
215
if self.file and not self.ignoreErrors:
216
self.client.transport.loseConnection()
219
def cmd_CD(self, path):
220
path, rest = self._getFilename(path)
221
if not path.endswith('/'):
223
newPath = path and os.path.join(self.currentDirectory, path) or ''
224
d = self.client.openDirectory(newPath)
225
d.addCallback(self._cbCd)
226
d.addErrback(self._ebCommand)
229
def _cbCd(self, directory):
231
d = self.client.realPath(directory.name)
232
d.addCallback(self._cbCurDir)
235
def _cbCurDir(self, path):
236
self.currentDirectory = path
238
def cmd_CHGRP(self, rest):
239
grp, rest = rest.split(None, 1)
240
path, rest = self._getFilename(rest)
242
d = self.client.getAttrs(path)
243
d.addCallback(self._cbSetUsrGrp, path, grp=grp)
246
def cmd_CHMOD(self, rest):
247
mod, rest = rest.split(None, 1)
248
path, rest = self._getFilename(rest)
250
d = self.client.setAttrs(path, {'permissions':mod})
251
d.addCallback(_ignore)
254
def cmd_CHOWN(self, rest):
255
usr, rest = rest.split(None, 1)
256
path, rest = self._getFilename(rest)
258
d = self.client.getAttrs(path)
259
d.addCallback(self._cbSetUsrGrp, path, usr=usr)
262
def _cbSetUsrGrp(self, attrs, path, usr=None, grp=None):
264
new['uid'] = (usr is not None) and usr or attrs['uid']
265
new['gid'] = (grp is not None) and grp or attrs['gid']
266
d = self.client.setAttrs(path, new)
267
d.addCallback(_ignore)
270
def cmd_GET(self, rest):
271
remote, rest = self._getFilename(rest)
272
if '*' in remote or '?' in remote: # wildcard
274
local, rest = self._getFilename(rest)
275
if not os.path.isdir(local):
276
return "Wildcard get with non-directory target."
279
d = self._remoteGlob(remote)
280
d.addCallback(self._cbGetMultiple, local)
283
local, rest = self._getFilename(rest)
285
local = os.path.split(remote)[1]
286
log.msg((remote, local))
287
lf = file(local, 'w', 0)
288
path = os.path.join(self.currentDirectory, remote)
289
d = self.client.openFile(path, filetransfer.FXF_READ, {})
290
d.addCallback(self._cbGetOpenFile, lf)
291
d.addErrback(self._ebCloseLf, lf)
294
def _cbGetMultiple(self, files, local):
295
#if self._useProgressBar: # one at a time
296
# XXX this can be optimized for times w/o progress bar
297
return self._cbGetMultipleNext(None, files, local)
299
def _cbGetMultipleNext(self, res, files, local):
300
if isinstance(res, failure.Failure):
301
self._printFailure(res)
303
self.transport.write(res)
304
if not res.endswith('\n'):
305
self.transport.write('\n')
309
lf = file(os.path.join(local, os.path.split(f)[1]), 'w', 0)
310
path = os.path.join(self.currentDirectory, f)
311
d = self.client.openFile(path, filetransfer.FXF_READ, {})
312
d.addCallback(self._cbGetOpenFile, lf)
313
d.addErrback(self._ebCloseLf, lf)
314
d.addBoth(self._cbGetMultipleNext, files, local)
317
def _ebCloseLf(self, f, lf):
321
def _cbGetOpenFile(self, rf, lf):
322
return rf.getAttrs().addCallback(self._cbGetFileSize, rf, lf)
324
def _cbGetFileSize(self, attrs, rf, lf):
325
if not stat.S_ISREG(attrs['permissions']):
328
return "Can't get non-regular file: %s" % rf.name
329
rf.size = attrs['size']
330
bufferSize = self.client.transport.conn.options['buffersize']
331
numRequests = self.client.transport.conn.options['requests']
335
startTime = time.time()
336
for i in range(numRequests):
337
d = self._cbGetRead('', rf, lf, chunks, 0, bufferSize, startTime)
339
dl = defer.DeferredList(dList, fireOnOneErrback=1)
340
dl.addCallback(self._cbGetDone, rf, lf)
343
def _getNextChunk(self, chunks):
347
return # nothing more to get
349
i = chunks.index(chunk)
350
chunks.insert(i, (end, chunk[0]))
351
return (end, chunk[0] - end)
353
bufSize = int(self.client.transport.conn.options['buffersize'])
354
chunks.append((end, end + bufSize))
355
return (end, bufSize)
357
def _cbGetRead(self, data, rf, lf, chunks, start, size, startTime):
358
if data and isinstance(data, failure.Failure):
359
log.msg('get read err: %s' % data)
361
reason.trap(EOFError)
362
i = chunks.index((start, start + size))
364
chunks.insert(i, (start, 'eof'))
366
log.msg('get read data: %i' % len(data))
369
if len(data) != size:
370
log.msg('got less than we asked for: %i < %i' %
372
i = chunks.index((start, start + size))
374
chunks.insert(i, (start, start + len(data)))
375
rf.total += len(data)
376
if self.useProgressBar:
377
self._printProgessBar(rf, startTime)
378
chunk = self._getNextChunk(chunks)
382
start, length = chunk
383
log.msg('asking for %i -> %i' % (start, start+length))
384
d = rf.readChunk(start, length)
385
d.addBoth(self._cbGetRead, rf, lf, chunks, start, length, startTime)
388
def _cbGetDone(self, ignored, rf, lf):
392
if self.useProgressBar:
393
self.transport.write('\n')
394
return "Transferred %s to %s" % (rf.name, lf.name)
396
def cmd_PUT(self, rest):
397
local, rest = self._getFilename(rest)
398
if '*' in local or '?' in local: # wildcard
400
remote, rest = self._getFilename(rest)
401
path = os.path.join(self.currentDirectory, remote)
402
d = self.client.getAttrs(path)
403
d.addCallback(self._cbPutTargetAttrs, remote, local)
407
files = glob.glob(local)
408
return self._cbPutMultipleNext(None, files, remote)
410
remote, rest = self._getFilename(rest)
412
remote = os.path.split(local)[1]
413
lf = file(local, 'r')
414
path = os.path.join(self.currentDirectory, remote)
415
d = self.client.openFile(path, filetransfer.FXF_WRITE|filetransfer.FXF_CREAT, {})
416
d.addCallback(self._cbPutOpenFile, lf)
417
d.addErrback(self._ebCloseLf, lf)
420
def _cbPutTargetAttrs(self, attrs, path, local):
421
if not stat.S_ISDIR(attrs['permissions']):
422
return "Wildcard put with non-directory target."
423
return self._cbPutMultipleNext(None, files, path)
425
def _cbPutMultipleNext(self, res, files, path):
426
if isinstance(res, failure.Failure):
427
self._printFailure(res)
429
self.transport.write(res)
430
if not res.endswith('\n'):
431
self.transport.write('\n')
433
while files and not f:
438
self._printFailure(failure.Failure())
442
name = os.path.split(f)[1]
443
remote = os.path.join(self.currentDirectory, path, name)
444
log.msg((name, remote, path))
445
d = self.client.openFile(remote, filetransfer.FXF_WRITE|filetransfer.FXF_CREAT, {})
446
d.addCallback(self._cbPutOpenFile, lf)
447
d.addErrback(self._ebCloseLf, lf)
448
d.addBoth(self._cbPutMultipleNext, files, path)
451
def _cbPutOpenFile(self, rf, lf):
452
numRequests = self.client.transport.conn.options['requests']
453
if self.useProgressBar:
457
startTime = time.time()
458
for i in range(numRequests):
459
d = self._cbPutWrite(None, rf, lf, chunks, startTime)
462
dl = defer.DeferredList(dList, fireOnOneErrback=1)
463
dl.addCallback(self._cbPutDone, rf, lf)
466
def _cbPutWrite(self, ignored, rf, lf, chunks, startTime):
467
chunk = self._getNextChunk(chunks)
471
if self.useProgressBar:
472
lf.total += len(data)
473
self._printProgessBar(lf, startTime)
475
d = rf.writeChunk(start, data)
476
d.addCallback(self._cbPutWrite, rf, lf, chunks, startTime)
481
def _cbPutDone(self, ignored, rf, lf):
484
if self.useProgressBar:
485
self.transport.write('\n')
486
return 'Transferred %s to %s' % (lf.name, rf.name)
488
def cmd_LCD(self, path):
491
def cmd_LN(self, rest):
492
linkpath, rest = self._getFilename(rest)
493
targetpath, rest = self._getFilename(rest)
494
linkpath, targetpath = map(
495
lambda x: os.path.join(self.currentDirectory, x),
496
(linkpath, targetpath))
497
return self.client.makeLink(linkpath, targetpath).addCallback(_ignore)
499
def cmd_LS(self, rest):
501
# ls current directory
502
# ls name_of_file that file
503
# ls name_of_directory that directory
504
# ls some_glob_string current directory, globbed for that string
507
while rest and rest[0] and rest[0][0] == '-':
508
opts = rest.pop(0)[1:]
511
options.append('verbose')
513
options.append('all')
514
rest = ' '.join(rest)
515
path, rest = self._getFilename(rest)
517
fullPath = self.currentDirectory + '/'
519
fullPath = os.path.join(self.currentDirectory, path)
520
d = self._remoteGlob(fullPath)
521
d.addCallback(self._cbDisplayFiles, options)
524
def _cbDisplayFiles(self, files, options):
526
if 'all' not in options:
527
files = [f for f in files if not f[0].startswith('.')]
528
if 'verbose' in options:
529
lines = [f[1] for f in files]
531
lines = [f[0] for f in files]
535
return '\n'.join(lines)
537
def cmd_MKDIR(self, path):
538
path, rest = self._getFilename(path)
539
path = os.path.join(self.currentDirectory, path)
540
return self.client.makeDirectory(path, {}).addCallback(_ignore)
542
def cmd_RMDIR(self, path):
543
path, rest = self._getFilename(path)
544
path = os.path.join(self.currentDirectory, path)
545
return self.client.removeDirectory(path).addCallback(_ignore)
547
def cmd_LMKDIR(self, path):
548
os.system("mkdir %s" % path)
550
def cmd_RM(self, path):
551
path, rest = self._getFilename(path)
552
path = os.path.join(self.currentDirectory, path)
553
return self.client.removeFile(path).addCallback(_ignore)
555
def cmd_LLS(self, rest):
556
os.system("ls %s" % rest)
558
def cmd_RENAME(self, rest):
559
oldpath, rest = self._getFilename(rest)
560
newpath, rest = self._getFilename(rest)
561
oldpath, newpath = map (
562
lambda x: os.path.join(self.currentDirectory, x),
564
return self.client.renameFile(oldpath, newpath).addCallback(_ignore)
566
def cmd_EXIT(self, ignored):
567
self.client.transport.loseConnection()
571
def cmd_VERSION(self, ignored):
572
return "SFTP version %i" % self.client.version
574
def cmd_HELP(self, ignored):
575
return """Available commands:
576
cd path Change remote directory to 'path'.
577
chgrp gid path Change gid of 'path' to 'gid'.
578
chmod mode path Change mode of 'path' to 'mode'.
579
chown uid path Change uid of 'path' to 'uid'.
580
exit Disconnect from the server.
581
get remote-path [local-path] Get remote file.
582
help Get a list of available commands.
583
lcd path Change local directory to 'path'.
584
lls [ls-options] [path] Display local directory listing.
585
lmkdir path Create local directory.
586
ln linkpath targetpath Symlink remote file.
587
lpwd Print the local working directory.
588
ls [-l] [path] Display remote directory listing.
589
mkdir path Create remote directory.
590
progress Toggle progress bar.
591
put local-path [remote-path] Put local file.
592
pwd Print the remote working directory.
593
quit Disconnect from the server.
594
rename oldpath newpath Rename remote file.
595
rmdir path Remove remote directory.
596
rm path Remove remote file.
597
version Print the SFTP version.
598
? Synonym for 'help'.
601
def cmd_PWD(self, ignored):
602
return self.currentDirectory
604
def cmd_LPWD(self, ignored):
607
def cmd_PROGRESS(self, ignored):
608
self.useProgressBar = not self.useProgressBar
609
return "%ssing progess bar." % (self.useProgressBar and "U" or "Not u")
611
def cmd_EXEC(self, rest):
612
shell = pwd.getpwnam(getpass.getuser())[6]
616
return utils.getProcessOutput(shell, cmds, errortoo=1)
620
# accessory functions
622
def _remoteGlob(self, fullPath):
623
log.msg('looking up %s' % fullPath)
624
head, tail = os.path.split(fullPath)
625
if '*' in tail or '?' in tail:
629
if tail and not glob: # could be file or directory
630
# try directory first
631
d = self.client.openDirectory(fullPath)
632
d.addCallback(self._cbOpenList, '')
633
d.addErrback(self._ebNotADirectory, head, tail)
635
d = self.client.openDirectory(head)
636
d.addCallback(self._cbOpenList, tail)
639
def _cbOpenList(self, directory, glob):
642
d.addBoth(self._cbReadFile, files, directory, glob)
645
def _ebNotADirectory(self, reason, path, glob):
646
d = self.client.openDirectory(path)
647
d.addCallback(self._cbOpenList, glob)
650
def _cbReadFile(self, files, l, directory, glob):
651
if not isinstance(files, failure.Failure):
653
l.extend([f for f in files if fnmatch.fnmatch(f[0], glob)])
657
d.addBoth(self._cbReadFile, l, directory, glob)
661
reason.trap(EOFError)
665
def _abbrevSize(self, size):
666
# from http://mail.python.org/pipermail/python-list/1999-December/018395.html
676
for factor, suffix in _abbrevs:
679
return '%.1f' % (size/factor) + suffix
681
def _abbrevTime(self, t):
682
if t > 3600: # 1 hour
683
hours = int(t / 3600)
687
return "%i:%02i:%02i" % (hours, mins, t)
691
return "%02i:%02i" % (mins, t)
693
def _printProgessBar(self, f, startTime):
694
diff = time.time() - startTime
697
winSize = struct.unpack('4H',
698
fcntl.ioctl(0, tty.TIOCGWINSZ, '12345679'))
703
timeLeft = (f.size - total) / speed
707
back = '%3i%% %s %sps %s ' % ((total/f.size)*100, self._abbrevSize(total),
708
self._abbrevSize(total/diff), self._abbrevTime(timeLeft))
709
spaces = (winSize[1] - (len(front) + len(back) + 1)) * ' '
710
self.transport.write('\r%s%s%s' % (front, spaces, back))
712
def _getFilename(self, line):
720
for i in range(1,len(line)):
723
return ''.join(ret), ''.join(line[i+1:]).lstrip()
724
elif c == '\\': # quoted character
726
if line[i] not in '\'"\\':
727
raise IndexError, "bad quote: \\%s" % line[i]
732
raise IndexError, "unterminated quote"
733
ret = line.split(None, 1)
739
StdioClient.__dict__['cmd_?'] = StdioClient.cmd_HELP
741
class SSHConnection(connection.SSHConnection):
742
def serviceStarted(self):
743
self.openChannel(SSHSession())
745
class SSHSession(channel.SSHChannel):
749
def channelOpen(self, foo):
750
log.msg('session %s open' % self.id)
751
if self.conn.options['subsystem'].startswith('/'):
754
request = 'subsystem'
755
d = self.conn.sendRequest(self, request, \
756
common.NS(self.conn.options['subsystem']), wantReply=1)
757
d.addCallback(self._cbSubsystem)
758
d.addErrback(_ebExit)
760
def _cbSubsystem(self, result):
761
self.client = filetransfer.FileTransferClient()
762
self.client.makeConnection(self)
763
self.dataReceived = self.client.dataReceived
765
if self.conn.options['batchfile']:
766
fn = self.conn.options['batchfile']
769
self.stdio = stdio.StandardIO(StdioClient(self.client, f))
771
def extReceived(self, t, data):
772
if t==connection.EXTENDED_DATA_STDERR:
773
log.msg('got %s stderr data' % len(data))
774
sys.stderr.write(data)
777
def eofReceived(self):
779
self.stdio.closeStdin()
781
def closeReceived(self):
782
log.msg('remote side closed %s' % self)
783
self.conn.sendClose(self)
791
def stopWriting(self):
792
self.stdio.pauseProducing()
794
def startWriting(self):
795
self.stdio.resumeProducing()
797
if __name__ == '__main__':