2
# client.py - Max Usachev <maxusachev@gmail.com>
3
# Ed Bartosh <bartosh@gmail.com>
4
# Peter Bienstman <Peter.Bienstman@UGent.be>
13
from partner import Partner
14
from text_formats.xml_format import XMLFormat
15
from utils import traceback_string, SyncError, SeriousSyncError
17
# Avoid delays caused by Nagle's algorithm.
18
# http://www.cmlenz.net/archives/2008/03/python-httplib-performance-problems
20
realsocket = socket.socket
21
def socketwrap(family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0):
22
sockobj = realsocket(family, type, proto)
23
sockobj.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
25
socket.socket = socketwrap
27
# Buffer the response socket.
28
# http://mail.python.org/pipermail/python-bugs-list/2006-September/035156.html
29
# Fix included here for systems running Python 2.5.
31
class HTTPResponse(httplib.HTTPResponse):
33
def __init__(self, sock, **kw):
34
httplib.HTTPResponse.__init__(self, sock, **kw)
35
self.fp = sock.makefile("rb") # Was unbuffered: sock.makefile("rb", 0)
37
httplib.HTTPConnection.response_class = HTTPResponse
39
# Register binary formats.
41
from binary_formats.mnemosyne_format import MnemosyneFormat
42
BinaryFormats = [MnemosyneFormat]
45
class Client(Partner):
47
program_name = "unknown-SRS-app"
48
program_version = "unknown"
50
# The capabilities supported by the client. Note that we assume that the
51
# server supports "mnemosyne_dynamic_cards".
52
capabilities = "mnemosyne_dynamic_cards" # "facts", "cards"
53
# The following setting can be set to False to speed up the syncing
54
# process on e.g. mobile clients where the media files don't get edited
56
check_for_edited_local_media_files = True
57
# Setting the following to False will speed up the initial sync, but in that
58
# case the client will not have access to all of the review history in order
59
# to e.g. display statistics. Also, it will not be possible to keep the
60
# local database when there are sync conflicts. If a client makes it
61
# possible for the user to change this value, doing so should result in
62
# redownloading the entire database from scratch.
63
interested_in_old_reps = True
64
# Store prerendered question, answer and tag fields in database. The only
65
# benefit of this is fast operation for a 'browse cards' dialog which
66
# directly operates at the SQL level. If you don't use this, set to False
67
# to reduce the database size.
68
store_pregenerated_data = True
69
# On mobile clients with slow SD cards copying a large database for the
70
# backup before sync can take longer than the sync itself, so we offer
71
# reckless users the possibility to skip this.
73
# Setting this to False will leave all the uploading of anonymous science
74
# logs to the sync server. Recommended to set this to False for mobile
75
# clients, which are not always guaranteed to have an internet connection.
76
upload_science_logs = True
78
def __init__(self, machine_id, database, ui):
79
Partner.__init__(self, ui)
80
self.machine_id = machine_id
81
self.database = database
82
self.text_format = XMLFormat()
85
self.behind_proxy = None # Explicit variable for testability.
88
def request_connection(self):
90
"""If we are not behind a proxy, create the connection once and reuse
91
it for all requests. If we are behind a proxy, we need to revert to
92
HTTP 1.0 and use a separate connection for each request.
96
# If we haven't done so, determine whether we're behind a proxy.
97
if self.behind_proxy is None:
99
proxies = urllib.getproxies()
100
if "http" in proxies:
101
self.behind_proxy = True
102
self.proxy = proxies["http"]
104
self.behind_proxy = False
105
# Create a new connection or reuse an existing one.
106
if self.behind_proxy:
107
httplib.HTTPConnection._http_vsn = 10
108
httplib.HTTPConnection._http_vsn_str = "HTTP/1.0"
109
if self.proxy is not None:
110
self.con = httplib.HTTPConnection(self.proxy, self.port)
111
else: # Testsuite has set self.behind_proxy to True to simulate
112
# being behind a proxy.
113
self.con = httplib.HTTPConnection(self.server, self.port)
115
httplib.HTTPConnection._http_vsn = 11
116
httplib.HTTPConnection._http_vsn_str = "HTTP/1.1"
118
self.con = httplib.HTTPConnection(self.server, self.port)
120
def url(self, url_string):
121
if self.behind_proxy and self.proxy:
122
url_string = self.server + ":/" + url_string
125
def sync(self, server, port, username, password):
127
self.server = socket.gethostbyname(server)
130
self.ui.set_progress_text("Creating backup...")
131
backup_file = self.database.backup()
132
# We check if files were edited outside of the program. This can
133
# generate EDITED_MEDIA_FILES log entries, so it should be done
135
if self.check_for_edited_local_media_files:
136
self.ui.set_progress_text("Checking for edited media files...")
137
self.database.check_for_edited_media_files()
138
self.ui.set_progress_text("Dynamically creating media files...")
139
self.database.dynamically_create_media_files()
140
# Set timeout long enough for e.g. a slow NAS waking from
142
socket.setdefaulttimeout(45)
143
self.login(username, password)
144
# Generating media files at the server side could take some time,
145
# so we update the timeout.
147
socket.setdefaulttimeout(15*60)
148
self.get_server_check_media_files()
149
# Do a full sync after either the client or the server has restored
151
if self.database.is_sync_reset_needed(\
152
self.server_info["machine_id"]) or \
153
self.server_info["sync_reset_needed"] == True:
154
self.resolve_conflicts(restored_from_backup=True)
155
# First sync, fetch database from server.
156
elif self.database.is_empty():
157
self.get_server_media_files()
158
if self.server_info["supports_binary_transfer"]:
159
self.get_server_entire_database_binary()
161
self.get_server_entire_database()
162
self.get_sync_finish()
163
# Fetch config settings.
164
self.login(username, password)
165
self.get_server_generate_log_entries_for_settings()
166
self.get_server_log_entries()
167
self.get_sync_finish()
168
# First sync, put binary database to server if supported.
169
elif not self.database.is_empty() and \
170
self.server_info["is_database_empty"] and \
171
self.supports_binary_upload():
172
self.put_client_media_files(reupload_all=True)
173
self.put_client_entire_database_binary()
174
self.get_sync_finish()
175
# Upload config settings.
176
self.login(username, password)
177
self.database.generate_log_entries_for_settings()
178
self.put_client_log_entries()
179
self.get_server_log_entries()
180
self.get_sync_finish()
182
# Upload local changes and check for conflicts.
183
result = self.put_client_log_entries()
185
self.put_client_media_files()
186
self.get_server_media_files()
187
self.get_server_log_entries()
188
self.get_sync_finish()
190
self.resolve_conflicts()
191
self.ui.show_information("Sync finished!")
192
except Exception, exception:
193
self.ui.close_progress()
195
if type(exception) == type(socket.gaierror()):
196
self.ui.show_error("Could not find server!")
198
elif type(exception) == type(socket.error()):
199
self.ui.show_error("Could not connect to server!")
201
elif type(exception) == type(socket.timeout()):
202
self.ui.show_error("Timeout while waiting for server!")
203
elif type(exception) == type(SyncError()):
204
self.ui.show_error(str(exception))
206
elif type(exception) == type(SeriousSyncError()):
207
self.ui.show_error(str(exception))
209
self.ui.show_error(traceback_string())
210
if serious and self.do_backup:
211
# Only serious errors should result in the need for a full
213
self.ui.show_error("Sync failed, restoring from backup. " + \
214
"The next sync will need to be a full sync.")
216
self.database.restore(backup_file)
220
self.ui.close_progress()
222
def supports_binary_upload(self):
223
return self.capabilities == "mnemosyne_dynamic_cards" and \
224
self.interested_in_old_reps and self.store_pregenerated_data \
225
and self.program_name == self.server_info["program_name"] and \
226
self.program_version == self.server_info["program_version"]
228
def resolve_conflicts(self, restored_from_backup=False):
229
if restored_from_backup:
230
message = "The database was restored from a backup, either " + \
231
"automatically because of an aborted sync or manually by " + \
232
"the user.\nFor safety, a full sync needs to happen and " + \
233
"you need to choose which copy of the database to keep and " + \
234
"which copy to discard.\n"
236
message = "Conflicts detected during sync! Choose which version to keep and which version to discard."
237
# Ask for conflict resolution direction.
238
if self.supports_binary_upload():
239
result = self.ui.show_question(message,
240
"Keep local version", "Fetch remote version", "Cancel")
241
results = {0: "KEEP_LOCAL", 1: "KEEP_REMOTE", 2: "CANCEL"}
242
result = results[result]
244
message += " " + "Your client only stores part of the server " + \
245
"database or uses a different software version, " + \
246
"so you can only fetch the remote version."
247
result = self.ui.show_question(message,
248
"Fetch remote version", "Cancel", "")
249
results = {0: "KEEP_REMOTE", 1: "CANCEL"}
250
result = results[result]
251
# Keep remote. Reset the partnerships afterwards, such that syncing
252
# with a third party will also trigger a full sync.
253
if result == "KEEP_REMOTE":
254
self.get_server_media_files(redownload_all=True)
255
if self.server_info["supports_binary_transfer"]:
256
self.get_server_entire_database_binary()
258
self.get_server_entire_database()
259
self.database.reset_partnerships()
260
self.get_sync_finish()
262
elif result == "KEEP_LOCAL":
263
self.put_client_media_files(reupload_all=True)
264
self.put_client_entire_database_binary()
265
self.get_sync_finish()
267
elif result == "CANCEL":
268
self.get_sync_cancel()
270
def _check_response_for_errors(self, response, can_consume_response=True):
271
# Check for non-Mnemosyne error messages.
272
if response.status != httplib.OK:
273
raise SeriousSyncError("Internal server error:\n" + response.read())
274
if can_consume_response == False:
276
# Check for Mnemosyne error messages.
277
message, traceback = self.text_format.parse_message(response.read())
278
if "server error" in message.lower():
279
raise SeriousSyncError(message + "\n" + traceback)
281
def login(self, username, password):
282
self.ui.set_progress_text("Logging in...")
284
client_info["username"] = username
285
client_info["password"] = password
286
client_info["user_id"] = self.database.user_id()
287
client_info["machine_id"] = self.machine_id
288
client_info["program_name"] = self.program_name
289
client_info["program_version"] = self.program_version
290
client_info["database_name"] = self.database.name()
291
client_info["database_version"] = self.database.version
292
client_info["capabilities"] = self.capabilities
293
client_info["partners"] = self.database.partners()
294
client_info["interested_in_old_reps"] = self.interested_in_old_reps
295
client_info["store_pregenerated_data"] = self.store_pregenerated_data
296
client_info["upload_science_logs"] = self.upload_science_logs
297
# Signal if the database is empty, so that the server does not give a
298
# spurious sync cycle warning if the client database was reset.
299
client_info["is_database_empty"] = self.database.is_empty()
300
# Not yet implemented: preferred renderer.
301
client_info["render_chain"] = ""
302
# Add optional program-specific information.
303
client_info = self.database.append_to_sync_partner_info(client_info)
304
# Try to establish a connection, but don't force a restore from backup
307
self.request_connection()
308
self.con.request("PUT", self.url("/login"),
309
self.text_format.repr_partner_info(client_info).\
310
encode("utf-8") + "\n")
311
response = self.con.getresponse()
313
raise SyncError("Could not connect to server!")
314
# Check for errors, but don't force a restore from backup if we can't
317
self._check_response_for_errors(\
318
response, can_consume_response=False)
319
except SeriousSyncError:
320
raise SyncError("Logging in: server error.")
321
response = response.read()
322
if "message" in response:
323
message, traceback = self.text_format.parse_message(response)
324
message = message.lower()
325
if "server error" in message:
326
raise SyncError("Logging in: server error.")
327
if "access denied" in message:
328
raise SyncError("Wrong username or password.")
329
if "cycle" in message:
331
"Sync cycle detected. Sync through intermediate partner.")
332
if "same machine ids" in message:
334
"You have manually copied the data directory before sync. Sync needs to start from an empty database.")
335
self.server_info = self.text_format.parse_partner_info(response)
336
self.database.set_sync_partner_info(self.server_info)
337
if self.database.is_empty():
338
self.database.change_user_id(self.server_info["user_id"])
339
elif self.server_info["user_id"] != client_info["user_id"] and \
340
self.server_info["is_database_empty"] == False:
341
raise SyncError("Error: mismatched user ids.\n" + \
342
"The first sync should happen on an empty database.")
343
self.database.create_if_needed_partnership_with(\
344
self.server_info["machine_id"])
345
self.database.merge_partners(self.server_info["partners"])
347
def get_server_check_media_files(self):
348
self.ui.set_progress_text("Asking server to check for updated media files...")
349
self.request_connection()
350
self.con.request("GET", self.url(\
351
"/server_check_media_files?" + \
352
"session_token=%s" % (self.server_info["session_token"], )))
353
response = self.con.getresponse()
354
self._check_response_for_errors(response, can_consume_response=True)
356
def put_client_log_entries(self):
358
"""Contrary to binary files, the size of the log is not known until we
359
create it. In order to save memory on mobile devices, we don't want to
360
construct the entire log in memory before sending it on to the server.
361
However, chunked uploads are in the grey area of the WSGI spec and are
362
also not supported by older HTTP 1.0 proxies (e.g. Squid before 3.1).
363
Therefore, as a compromise, rather then streaming chunks in a single
364
message, we break up the entire log in different messages of size
369
number_of_entries = self.database.number_of_log_entries_to_sync_for(\
370
self.server_info["machine_id"])
371
if number_of_entries == 0:
373
self.ui.set_progress_text("Sending log entries...")
374
self.ui.set_progress_range(number_of_entries)
375
self.ui.set_progress_update_interval(number_of_entries/20)
378
for log_entry in self.database.log_entries_to_sync_for(\
379
self.server_info["machine_id"]):
380
buffer += self.text_format.repr_log_entry(log_entry)
382
self.ui.increase_progress(1)
383
if len(buffer) > self.BUFFER_SIZE or count == number_of_entries:
385
self.text_format.log_entries_header(number_of_entries) \
386
+ buffer + self.text_format.log_entries_footer()
387
self.request_connection()
388
self.con.request("PUT", self.url(\
389
"/client_log_entries?session_token=%s" \
390
% (self.server_info["session_token"],)),
391
buffer.encode("utf-8"))
393
response = self.con.getresponse()
394
self._check_response_for_errors(response,
395
can_consume_response=False)
396
response = response.read()
397
message, traceback = self.text_format.parse_message(response)
398
message = message.lower()
399
if "server error" in message:
400
raise SeriousSyncError(message)
401
if "conflict" in message:
405
def put_client_entire_database_binary(self):
406
self.ui.set_progress_text("Sending entire binary database...")
407
for BinaryFormat in BinaryFormats:
408
binary_format = BinaryFormat(self.database)
409
if binary_format.supports(self.server_info["program_name"],
410
self.server_info["program_version"],
411
self.server_info["database_version"]):
412
assert self.store_pregenerated_data == True
413
assert self.interested_in_old_reps == True
414
filename = binary_format.binary_filename(\
415
self.store_pregenerated_data, self.interested_in_old_reps)
417
self.request_connection()
418
self.con.putrequest("PUT",
419
self.url("/client_entire_database_binary?session_token=%s" \
420
% (self.server_info["session_token"], )))
421
self.con.putheader("content-length", os.path.getsize(filename))
422
self.con.endheaders()
423
for buffer in self.stream_binary_file(filename):
424
self.con.send(buffer)
425
binary_format.clean_up()
426
self._check_response_for_errors(self.con.getresponse())
428
def _download_log_entries(self, stream):
429
element_loop = self.text_format.parse_log_entries(stream)
430
number_of_entries = int(element_loop.next())
431
if number_of_entries == 0:
433
self.ui.set_progress_range(number_of_entries)
434
self.ui.set_progress_update_interval(number_of_entries/50)
435
for log_entry in element_loop:
436
self.database.apply_log_entry(log_entry)
437
self.ui.increase_progress(1)
438
self.ui.set_progress_value(number_of_entries)
440
def get_server_log_entries(self):
441
self.ui.set_progress_text("Getting log entries...")
442
if self.upload_science_logs:
443
self.database.dump_to_science_log()
444
self.request_connection()
445
self.con.request("GET", self.url(\
446
"/server_log_entries?session_token=%s" \
447
% (self.server_info["session_token"], )))
448
response = self.con.getresponse()
449
self._check_response_for_errors(response, can_consume_response=False)
450
self._download_log_entries(response)
451
# The server will always upload the science logs of the log events
452
# which originated at the server side.
453
self.database.skip_science_log()
455
def get_server_entire_database(self):
456
self.ui.set_progress_text("Getting entire database...")
457
filename = self.database.path()
458
# Create a new database. Note that this also resets the
459
# partnerships, as required.
460
self.database.new(filename)
461
self.request_connection()
462
self.con.request("GET", self.url("/server_entire_database?" + \
463
"session_token=%s" % (self.server_info["session_token"], )))
464
response = self.con.getresponse()
465
self._check_response_for_errors(response, can_consume_response=False)
466
self._download_log_entries(response)
467
self.database.load(filename)
468
# The server will always upload the science logs of the log events
469
# which originated at the server side.
470
self.database.skip_science_log()
471
# Since we start from a new database, we need to create the
473
self.database.create_if_needed_partnership_with(\
474
self.server_info["machine_id"])
476
def get_server_entire_database_binary(self):
477
self.ui.set_progress_text("Getting entire binary database...")
478
filename = self.database.path()
479
self.database.abandon()
480
self.request_connection()
481
self.con.request("GET", self.url(\
482
"/server_entire_database_binary?" + \
483
"session_token=%s" % (self.server_info["session_token"], )))
484
response = self.con.getresponse()
485
self._check_response_for_errors(response, can_consume_response=False)
486
file_size = int(response.getheader("mnemosyne-content-length"))
487
self.download_binary_file(response, filename, file_size)
488
self.database.load(filename)
489
self.database.create_if_needed_partnership_with(\
490
self.server_info["machine_id"])
491
self.database.remove_partnership_with(self.machine_id)
493
def get_server_generate_log_entries_for_settings(self):
494
self.ui.set_progress_text("Getting settings...")
495
self.request_connection()
496
self.con.request("GET", self.url(\
497
"/server_generate_log_entries_for_settings?" + \
498
"session_token=%s" % (self.server_info["session_token"], )))
499
response = self.con.getresponse()
500
self._check_response_for_errors(response, can_consume_response=True)
502
def put_client_media_files(self, reupload_all=False):
503
self.ui.set_progress_text("Sending media files...")
505
filenames = list(self.database.all_media_filenames())
507
filenames = list(self.database.media_filenames_to_sync_for(\
508
self.server_info["machine_id"]))
510
for filename in filenames:
511
total_size += os.path.getsize(os.path.join(\
512
self.database.media_dir(), filename))
513
self.ui.set_progress_range(total_size)
514
self.ui.set_progress_update_interval(total_size/50)
515
for filename in filenames:
516
self.request_connection()
517
self.con.putrequest("PUT",
518
self.url("/client_media_file?session_token=%s&filename=%s" \
519
% (self.server_info["session_token"],
520
urllib.quote(filename.encode("utf-8"), ""))))
521
full_path = os.path.join(self.database.media_dir(), filename)
522
file_size = os.path.getsize(full_path)
523
self.con.putheader("content-length", file_size)
524
self.con.endheaders()
525
for buffer in self.stream_binary_file(full_path, progress_bar=False):
526
self.con.send(buffer)
527
self.ui.increase_progress(len(buffer))
528
self._check_response_for_errors(self.con.getresponse())
529
self.ui.set_progress_value(total_size)
531
def get_server_media_files(self, redownload_all=False):
532
self.ui.set_progress_text("Getting list of media files to download...")
533
# Get list of names of all media files to download.
534
media_url = "/server_media_filenames?session_token=%s" \
535
% (self.server_info["session_token"], )
537
media_url += "&redownload_all=1"
538
self.request_connection()
539
self.con.request("GET", self.url(media_url))
540
response = self.con.getresponse()
541
self._check_response_for_errors(response, can_consume_response=False)
542
total_size = int(response.getheader("mnemosyne-content-length"))
544
# Make sure to read the full message, even if it's empty,
545
# since we reuse our connection.
548
# Download each media file.
549
self.ui.set_progress_text("Getting media files...")
550
self.ui.set_progress_range(total_size)
551
self.ui.set_progress_update_interval(total_size/50)
552
for filename in response.read().split("\n"):
553
filename = unicode(filename, "utf-8")
554
self.request_connection()
555
self.con.request("GET",
556
self.url("/server_media_file?session_token=%s&filename=%s" \
557
% (self.server_info["session_token"],
558
urllib.quote(filename.encode("utf-8"), ""))))
559
response = self.con.getresponse()
560
self._check_response_for_errors(response,
561
can_consume_response=False)
562
file_size = int(response.getheader("mnemosyne-content-length"))
563
# Make sure a malicious server cannot overwrite anything outside
564
# of the media directory.
565
filename = filename.replace("../", "").replace("..\\", "")
566
filename = filename.replace("/..", "").replace("\\..", "")
567
filename = os.path.join(self.database.media_dir(), filename)
568
self.download_binary_file(response, filename,
569
file_size, progress_bar=False)
570
self.ui.increase_progress(file_size)
571
self.ui.set_progress_value(total_size)
573
def get_sync_cancel(self):
574
self.ui.set_progress_text("Cancelling sync...")
575
self.request_connection()
576
self.con.request("GET", self.url("/sync_cancel?session_token=%s" \
577
% (self.server_info["session_token"], )),
578
headers={"connection": "close"})
579
self._check_response_for_errors(self.con.getresponse())
581
def get_sync_finish(self):
582
self.ui.set_progress_text("Finishing sync...")
583
self.request_connection()
584
self.con.request("GET", self.url("/sync_finish?session_token=%s" \
585
% (self.server_info["session_token"], )),
586
headers={"connection": "close"})
587
self._check_response_for_errors(self.con.getresponse())
588
# Only update after we are sure there have been no errors.
589
self.database.update_last_log_index_synced_for(\
590
self.server_info["machine_id"])
added
removed
mnemosyne/openSM2sync/client.py
