← Back to branch summary

~jflaker/duplicity/BugFix1325215

« back to all changes in this revision

Viewing changes to duplicity/backends/_ssh_pexpect.py

  • Committer: Kenneth Loafman
  • Date: 2014-04-29 15:35:47 UTC
  • mfrom: (978.2.10 backend-unification)
  • Revision ID: kenneth@loafman.com-20140429153547-to2j1tyyl0ps1hi6
* Merged in lp:~mterry/duplicity/backend-unification
  - Reorganize and simplify backend code.  Specifically:
    - Formalize the expected API between backends and duplicity.  See the new
      file duplicity/backends/README for the instructions I've given authors.
    - Add some tests for our backend wrapper class as well as some tests for
      individual backends.  For several backends that have some commands do all
      the heavy lifting (hsi, tahoe, ftp), I've added fake little mock commands
      so that we can test them locally.  This doesn't truly test our integration
      with those commands, but at least lets us test the backend glue code.
    - Removed a lot of duplicate and unused code which backends were using (or
      not using).  This branch drops 700 lines of code (~20%)
      in duplicity/backends!
    - Simplified expectations of backends.  Our wrapper code now does all the
      retrying, and all the exception handling.  Backends can 'fire and forget'
      trusting our wrappers to give the user a reasonable error message.
      Obviously, backends can also add more details and make nicer error
      messages.  But they don't *have* to.
    - Separate out the backend classes from our wrapper class.  Now there is no
      possibility of namespace collision.  All our API methods use one
      underscore.  Anything else (zero or two underscores) are for the backend
      class's use.
    - Added the concept of a 'backend prefix' which is used by par2 and gio
      backends to provide generic support for "schema+" in urls -- like par2+
      or gio+.  I've since marked the '--gio' flag as deprecated, in favor of
      'gio+'.  Now you can even nest such backends like
      par2+gio+file://blah/blah.
    - The switch to control which cloudfiles backend had a typo.  I fixed this,
      but I'm not sure I should have?  If we haven't had complaints, maybe we
      can just drop the old backend.
    - I manually tested all the backends we have (except hsi and tahoe -- but
      those are simple wrappers around commands and I did test those via mocks
      per above).  I also added a bunch more manual backend tests to
      ./testing/manual/backendtest.py, which can now be run like the above to
      test all the files you have configured in config.py or you can pass it a
      URL which it will use for testing (useful for backend authors).

Show diffs side-by-side

added added

removed removed

Lines of Context:
duplicity/backends/_ssh_pexpect.py
24
24
# have the same syntax.  Also these strings will be executed by the
25
25
# shell, so shouldn't have strange characters in them.
26
26
 
 
27
from future_builtins import map
 
28
 
27
29
import re
28
30
import string
29
 
import time
30
31
import os
31
32
 
32
33
import duplicity.backend
33
34
from duplicity import globals
34
35
from duplicity import log
35
 
from duplicity.errors import * #@UnusedWildImport
 
36
from duplicity.errors import BackendException
36
37
 
37
38
class SSHPExpectBackend(duplicity.backend.Backend):
38
 
    """This backend copies files using scp.  List not supported"""
 
39
    """This backend copies files using scp.  List not supported.  Filenames
 
40
       should not need any quoting or this will break."""
39
41
    def __init__(self, parsed_url):
40
42
        """scpBackend initializer"""
41
43
        duplicity.backend.Backend.__init__(self, parsed_url)
76
78
    def run_scp_command(self, commandline):
77
79
        """ Run an scp command, responding to password prompts """
78
80
        import pexpect
79
 
        for n in range(1, globals.num_retries+1):
80
 
            if n > 1:
81
 
                # sleep before retry
82
 
                time.sleep(self.retry_delay)
83
 
            log.Info("Running '%s' (attempt #%d)" % (commandline, n))
84
 
            child = pexpect.spawn(commandline, timeout = None)
85
 
            if globals.ssh_askpass:
86
 
                state = "authorizing"
87
 
            else:
88
 
                state = "copying"
89
 
            while 1:
90
 
                if state == "authorizing":
91
 
                    match = child.expect([pexpect.EOF,
92
 
                                          "(?i)timeout, server not responding",
93
 
                                          "(?i)pass(word|phrase .*):",
94
 
                                          "(?i)permission denied",
95
 
                                          "authenticity"])
96
 
                    log.Debug("State = %s, Before = '%s'" % (state, child.before.strip()))
97
 
                    if match == 0:
98
 
                        log.Warn("Failed to authenticate")
99
 
                        break
100
 
                    elif match == 1:
101
 
                        log.Warn("Timeout waiting to authenticate")
102
 
                        break
103
 
                    elif match == 2:
104
 
                        child.sendline(self.password)
105
 
                        state = "copying"
106
 
                    elif match == 3:
107
 
                        log.Warn("Invalid SSH password")
108
 
                        break
109
 
                    elif match == 4:
110
 
                        log.Warn("Remote host authentication failed (missing known_hosts entry?)")
111
 
                        break
112
 
                elif state == "copying":
113
 
                    match = child.expect([pexpect.EOF,
114
 
                                          "(?i)timeout, server not responding",
115
 
                                          "stalled",
116
 
                                          "authenticity",
117
 
                                          "ETA"])
118
 
                    log.Debug("State = %s, Before = '%s'" % (state, child.before.strip()))
119
 
                    if match == 0:
120
 
                        break
121
 
                    elif match == 1:
122
 
                        log.Warn("Timeout waiting for response")
123
 
                        break
124
 
                    elif match == 2:
125
 
                        state = "stalled"
126
 
                    elif match == 3:
127
 
                        log.Warn("Remote host authentication failed (missing known_hosts entry?)")
128
 
                        break
129
 
                elif state == "stalled":
130
 
                    match = child.expect([pexpect.EOF,
131
 
                                          "(?i)timeout, server not responding",
132
 
                                          "ETA"])
133
 
                    log.Debug("State = %s, Before = '%s'" % (state, child.before.strip()))
134
 
                    if match == 0:
135
 
                        break
136
 
                    elif match == 1:
137
 
                        log.Warn("Stalled for too long, aborted copy")
138
 
                        break
139
 
                    elif match == 2:
140
 
                        state = "copying"
141
 
            child.close(force = True)
142
 
            if child.exitstatus == 0:
143
 
                return
144
 
            log.Warn("Running '%s' failed (attempt #%d)" % (commandline, n))
145
 
        log.Warn("Giving up trying to execute '%s' after %d attempts" % (commandline, globals.num_retries))
146
 
        raise BackendException("Error running '%s'" % commandline)
 
81
        log.Info("Running '%s'" % commandline)
 
82
        child = pexpect.spawn(commandline, timeout = None)
 
83
        if globals.ssh_askpass:
 
84
            state = "authorizing"
 
85
        else:
 
86
            state = "copying"
 
87
        while 1:
 
88
            if state == "authorizing":
 
89
                match = child.expect([pexpect.EOF,
 
90
                                      "(?i)timeout, server not responding",
 
91
                                      "(?i)pass(word|phrase .*):",
 
92
                                      "(?i)permission denied",
 
93
                                      "authenticity"])
 
94
                log.Debug("State = %s, Before = '%s'" % (state, child.before.strip()))
 
95
                if match == 0:
 
96
                    log.Warn("Failed to authenticate")
 
97
                    break
 
98
                elif match == 1:
 
99
                    log.Warn("Timeout waiting to authenticate")
 
100
                    break
 
101
                elif match == 2:
 
102
                    child.sendline(self.password)
 
103
                    state = "copying"
 
104
                elif match == 3:
 
105
                    log.Warn("Invalid SSH password")
 
106
                    break
 
107
                elif match == 4:
 
108
                    log.Warn("Remote host authentication failed (missing known_hosts entry?)")
 
109
                    break
 
110
            elif state == "copying":
 
111
                match = child.expect([pexpect.EOF,
 
112
                                      "(?i)timeout, server not responding",
 
113
                                      "stalled",
 
114
                                      "authenticity",
 
115
                                      "ETA"])
 
116
                log.Debug("State = %s, Before = '%s'" % (state, child.before.strip()))
 
117
                if match == 0:
 
118
                    break
 
119
                elif match == 1:
 
120
                    log.Warn("Timeout waiting for response")
 
121
                    break
 
122
                elif match == 2:
 
123
                    state = "stalled"
 
124
                elif match == 3:
 
125
                    log.Warn("Remote host authentication failed (missing known_hosts entry?)")
 
126
                    break
 
127
            elif state == "stalled":
 
128
                match = child.expect([pexpect.EOF,
 
129
                                      "(?i)timeout, server not responding",
 
130
                                      "ETA"])
 
131
                log.Debug("State = %s, Before = '%s'" % (state, child.before.strip()))
 
132
                if match == 0:
 
133
                    break
 
134
                elif match == 1:
 
135
                    log.Warn("Stalled for too long, aborted copy")
 
136
                    break
 
137
                elif match == 2:
 
138
                    state = "copying"
 
139
        child.close(force = True)
 
140
        if child.exitstatus != 0:
 
141
            raise BackendException("Error running '%s'" % commandline)
147
142
 
148
143
    def run_sftp_command(self, commandline, commands):
149
144
        """ Run an sftp command, responding to password prompts, passing commands from list """
160
155
                     "Couldn't delete file",
161
156
                     "open(.*): Failure"]
162
157
        max_response_len = max([len(p) for p in responses[1:]])
163
 
        for n in range(1, globals.num_retries+1):
164
 
            if n > 1:
165
 
                # sleep before retry
166
 
                time.sleep(self.retry_delay)
167
 
            log.Info("Running '%s' (attempt #%d)" % (commandline, n))
168
 
            child = pexpect.spawn(commandline, timeout = None, maxread=maxread)
169
 
            cmdloc = 0
170
 
            passprompt = 0
171
 
            while 1:
172
 
                msg = ""
173
 
                match = child.expect(responses,
174
 
                                     searchwindowsize=maxread+max_response_len)
175
 
                log.Debug("State = sftp, Before = '%s'" % (child.before.strip()))
176
 
                if match == 0:
177
 
                    break
178
 
                elif match == 1:
179
 
                    msg = "Timeout waiting for response"
180
 
                    break
181
 
                if match == 2:
182
 
                    if cmdloc < len(commands):
183
 
                        command = commands[cmdloc]
184
 
                        log.Info("sftp command: '%s'" % (command,))
185
 
                        child.sendline(command)
186
 
                        cmdloc += 1
187
 
                    else:
188
 
                        command = 'quit'
189
 
                        child.sendline(command)
190
 
                        res = child.before
191
 
                elif match == 3:
192
 
                    passprompt += 1
193
 
                    child.sendline(self.password)
194
 
                    if (passprompt>1):
195
 
                        raise BackendException("Invalid SSH password.")
196
 
                elif match == 4:
197
 
                    if not child.before.strip().startswith("mkdir"):
198
 
                        msg = "Permission denied"
199
 
                        break
200
 
                elif match == 5:
201
 
                    msg = "Host key authenticity could not be verified (missing known_hosts entry?)"
202
 
                    break
203
 
                elif match == 6:
204
 
                    if not child.before.strip().startswith("rm"):
205
 
                        msg = "Remote file or directory does not exist in command='%s'" % (commandline,)
206
 
                        break
207
 
                elif match == 7:
208
 
                    if not child.before.strip().startswith("Removing"):
209
 
                        msg = "Could not delete file in command='%s'" % (commandline,)
210
 
                        break;
211
 
                elif match == 8:
 
158
        log.Info("Running '%s'" % (commandline))
 
159
        child = pexpect.spawn(commandline, timeout = None, maxread=maxread)
 
160
        cmdloc = 0
 
161
        passprompt = 0
 
162
        while 1:
 
163
            msg = ""
 
164
            match = child.expect(responses,
 
165
                                 searchwindowsize=maxread+max_response_len)
 
166
            log.Debug("State = sftp, Before = '%s'" % (child.before.strip()))
 
167
            if match == 0:
 
168
                break
 
169
            elif match == 1:
 
170
                msg = "Timeout waiting for response"
 
171
                break
 
172
            if match == 2:
 
173
                if cmdloc < len(commands):
 
174
                    command = commands[cmdloc]
 
175
                    log.Info("sftp command: '%s'" % (command,))
 
176
                    child.sendline(command)
 
177
                    cmdloc += 1
 
178
                else:
 
179
                    command = 'quit'
 
180
                    child.sendline(command)
 
181
                    res = child.before
 
182
            elif match == 3:
 
183
                passprompt += 1
 
184
                child.sendline(self.password)
 
185
                if (passprompt>1):
 
186
                    raise BackendException("Invalid SSH password.")
 
187
            elif match == 4:
 
188
                if not child.before.strip().startswith("mkdir"):
 
189
                    msg = "Permission denied"
 
190
                    break
 
191
            elif match == 5:
 
192
                msg = "Host key authenticity could not be verified (missing known_hosts entry?)"
 
193
                break
 
194
            elif match == 6:
 
195
                if not child.before.strip().startswith("rm"):
 
196
                    msg = "Remote file or directory does not exist in command='%s'" % (commandline,)
 
197
                    break
 
198
            elif match == 7:
 
199
                if not child.before.strip().startswith("Removing"):
212
200
                    msg = "Could not delete file in command='%s'" % (commandline,)
213
 
                    break
214
 
                elif match == 9:
215
 
                    msg = "Could not open file in command='%s'" % (commandline,)
216
 
                    break
217
 
            child.close(force = True)
218
 
            if child.exitstatus == 0:
219
 
                return res
220
 
            log.Warn("Running '%s' with commands:\n %s\n failed (attempt #%d): %s" % (commandline, "\n ".join(commands), n, msg))
221
 
        raise BackendException("Giving up trying to execute '%s' with commands:\n %s\n after %d attempts" % (commandline, "\n ".join(commands), globals.num_retries))
 
201
                    break;
 
202
            elif match == 8:
 
203
                msg = "Could not delete file in command='%s'" % (commandline,)
 
204
                break
 
205
            elif match == 9:
 
206
                msg = "Could not open file in command='%s'" % (commandline,)
 
207
                break
 
208
        child.close(force = True)
 
209
        if child.exitstatus == 0:
 
210
            return res
 
211
        else:
 
212
            raise BackendException("Error running '%s': %s" % (commandline, msg))
222
213
 
223
 
    def put(self, source_path, remote_filename = None):
 
214
    def _put(self, source_path, remote_filename):
224
215
        if globals.use_scp:
225
 
            self.put_scp(source_path, remote_filename = remote_filename)
 
216
            self.put_scp(source_path, remote_filename)
226
217
        else:
227
 
            self.put_sftp(source_path, remote_filename = remote_filename)
 
218
            self.put_sftp(source_path, remote_filename)
228
219
 
229
 
    def put_sftp(self, source_path, remote_filename = None):
230
 
        """Use sftp to copy source_dir/filename to remote computer"""
231
 
        if not remote_filename:
232
 
            remote_filename = source_path.get_filename()
 
220
    def put_sftp(self, source_path, remote_filename):
233
221
        commands = ["put \"%s\" \"%s.%s.part\"" %
234
222
                    (source_path.name, self.remote_prefix, remote_filename),
235
223
                    "rename \"%s.%s.part\" \"%s%s\"" %
239
227
                                     self.host_string))
240
228
        self.run_sftp_command(commandline, commands)
241
229
 
242
 
    def put_scp(self, source_path, remote_filename = None):
243
 
        """Use scp to copy source_dir/filename to remote computer"""
244
 
        if not remote_filename:
245
 
            remote_filename = source_path.get_filename()
 
230
    def put_scp(self, source_path, remote_filename):
246
231
        commandline = "%s %s %s %s:%s%s" % \
247
232
            (self.scp_command, globals.ssh_options, source_path.name, self.host_string,
248
233
             self.remote_prefix, remote_filename)
249
234
        self.run_scp_command(commandline)
250
235
 
251
 
    def get(self, remote_filename, local_path):
 
236
    def _get(self, remote_filename, local_path):
252
237
        if globals.use_scp:
253
238
            self.get_scp(remote_filename, local_path)
254
239
        else:
255
240
            self.get_sftp(remote_filename, local_path)
256
241
 
257
242
    def get_sftp(self, remote_filename, local_path):
258
 
        """Use sftp to get a remote file"""
259
243
        commands = ["get \"%s%s\" \"%s\"" %
260
244
                    (self.remote_prefix, remote_filename, local_path.name)]
261
245
        commandline = ("%s %s %s" % (self.sftp_command,
262
246
                                     globals.ssh_options,
263
247
                                     self.host_string))
264
248
        self.run_sftp_command(commandline, commands)
265
 
        local_path.setdata()
266
 
        if not local_path.exists():
267
 
            raise BackendException("File %s not found locally after get "
268
 
                                   "from backend" % local_path.name)
269
249
 
270
250
    def get_scp(self, remote_filename, local_path):
271
 
        """Use scp to get a remote file"""
272
251
        commandline = "%s %s %s:%s%s %s" % \
273
252
            (self.scp_command, globals.ssh_options, self.host_string, self.remote_prefix,
274
253
             remote_filename, local_path.name)
275
254
        self.run_scp_command(commandline)
276
 
        local_path.setdata()
277
 
        if not local_path.exists():
278
 
            raise BackendException("File %s not found locally after get "
279
 
                                   "from backend" % local_path.name)
280
255
 
281
256
    def _list(self):
282
 
        """
283
 
        List files available for scp
284
 
 
285
 
        Note that this command can get confused when dealing with
286
 
        files with newlines in them, as the embedded newlines cannot
287
 
        be distinguished from the file boundaries.
288
 
        """
 
257
        # Note that this command can get confused when dealing with
 
258
        # files with newlines in them, as the embedded newlines cannot
 
259
        # be distinguished from the file boundaries.
289
260
        dirs = self.remote_dir.split(os.sep)
290
261
        if len(dirs) > 0:
291
262
            if not dirs[0] :
304
275
 
305
276
        return [x for x in map(string.strip, l) if x]
306
277
 
307
 
    def delete(self, filename_list):
308
 
        """
309
 
        Runs sftp rm to delete files.  Files must not require quoting.
310
 
        """
 
278
    def _delete(self, filename):
311
279
        commands = ["cd \"%s\"" % (self.remote_dir,)]
312
 
        for fn in filename_list:
313
 
            commands.append("rm \"%s\"" % fn)
 
280
        commands.append("rm \"%s\"" % filename)
314
281
        commandline = ("%s %s %s" % (self.sftp_command, globals.ssh_options, self.host_string))
315
282
        self.run_sftp_command(commandline, commands)
316
 
 
317
 
duplicity.backend.register_backend("ssh", SSHPExpectBackend)
318
 
duplicity.backend.register_backend("scp", SSHPExpectBackend)
319
 
duplicity.backend.register_backend("sftp", SSHPExpectBackend)