« back to all changes in this revision
Viewing changes to common/snapshots.py
- browse files at revision 588
- compare with another revision
- compare with revision 588
- download diff
- download tarball
- view history from revision 588
- Committer: Timo Kluck
- Date: 2009-11-26 22:25:01 UTC
- mfrom: (585.1.47 trunk)
- Revision ID: tkluck@netbook-tjk-20091126222501-6cfwq1jfud2c0m3j
* added kde support
* removed accidentally added ¨feature¨ of a per-folder on-mount action (added by copy pasting code)
* merge with current version in trunk
* removed accidentally added ¨feature¨ of a per-folder on-mount action (added by copy pasting code)
* merge with current version in trunk
- files removed:
- common/man/C/backintime.1.gz
- files modified:
- AUTHORS
added
removed
common/snapshots.py
1
1
# Back In Time
2
# Copyright (C) 2008-2009 Oprea Dan
2
# Copyright (C) 2008-2009 Oprea Dan, Bart de Koning, Richard Bailey
3
3
#
4
4
# This program is free software; you can redistribute it and/or modify
5
5
# it under the terms of the GNU General Public License as published by
26
26
import bz2
27
27
import pwd
28
28
import grp
29
import socket
29
30
30
31
import config
31
32
import configfile
39
40
40
41
41
42
class Snapshots:
43
SNAPSHOT_VERSION = 3
44
42
45
def __init__( self, cfg = None ):
43
46
self.config = cfg
44
47
if self.config is None:
47
50
self.plugin_manager = pluginmanager.PluginManager()
48
51
49
52
def get_snapshot_id( self, date ):
50
if type( date ) is datetime.datetime:
51
return date.strftime( '%Y%m%d-%H%M%S' )
52
53
if type( date ) is datetime.date:
54
return date.strftime( '%Y%m%d-000000' )
55
56
if type( date ) is str:
57
return date
53
profile_id = self.config.get_current_profile()
54
tag = self.config.get_tag( profile_id )
55
56
if type( date ) is datetime.datetime:
57
snapshot_id = date.strftime( '%Y%m%d-%H%M%S' ) + '-' + tag
58
return snapshot_id
59
60
if type( date ) is datetime.date:
61
snapshot_id = date.strftime( '%Y%m%d-000000' ) + '-' + tag
62
return snapshot_id
63
64
if type( date ) is str:
65
snapshot_id = date
66
return snapshot_id
67
68
return ""
69
70
def get_snapshot_old_id( self, date ):
71
profile_id = self.config.get_current_profile()
72
73
if type( date ) is datetime.datetime:
74
snapshot_id = date.strftime( '%Y%m%d-%H%M%S' )
75
return snapshot_id
76
77
if type( date ) is datetime.date:
78
snapshot_id = date.strftime( '%Y%m%d-000000' )
79
return snapshot_id
80
81
if type( date ) is str:
82
snapshot_id = date
83
return snapshot_id
58
84
59
85
return ""
60
86
61
87
def get_snapshot_path( self, date ):
62
return os.path.join( self.config.get_snapshots_full_path(), self.get_snapshot_id( date ) )
88
profile_id = self.config.get_current_profile()
89
path = os.path.join( self.config.get_snapshots_full_path( profile_id ), self.get_snapshot_id( date ) )
90
if os.path.exists( path ):
91
#print path
92
return path
93
other_folders = self.config.get_other_folders_paths()
94
for folder in other_folders:
95
path_other = os.path.join( folder, self.get_snapshot_id( date ) )
96
if os.path.exists( path_other ):
97
#print path_other
98
return path_other
99
old_path = os.path.join( self.config.get_snapshots_full_path( profile_id ), self.get_snapshot_old_id( date ) )
100
if os.path.exists( path ):
101
#print path
102
return path
103
other_folders = self.config.get_other_folders_paths()
104
for folder in other_folders:
105
path_other = os.path.join( folder, self.get_snapshot_old_id( date ) )
106
if os.path.exists( path_other ):
107
#print path_other
108
return path_other
109
#print path
110
return path
63
111
64
112
def get_snapshot_info_path( self, date ):
65
113
return os.path.join( self.get_snapshot_path( date ), 'info' )
256
304
257
305
backup_suffix = '.backup.' + datetime.date.today().strftime( '%Y%m%d' )
258
306
#cmd = "rsync -avR --copy-unsafe-links --whole-file --backup --suffix=%s --chmod=+w %s/.%s %s" % ( backup_suffix, self.get_snapshot_path_to( snapshot_id ), path, '/' )
259
cmd = "rsync -avRAXE --whole-file --backup --suffix=%s " % backup_suffix
260
cmd = cmd + '--chmod=+w '
307
cmd = "rsync -avRAXEH --whole-file --backup --suffix=%s " % backup_suffix
308
#cmd = cmd + '--chmod=+w '
261
309
cmd = cmd + "\"%s.%s\" %s" % ( self.get_snapshot_path_to( snapshot_id ), path, '/' )
262
310
self._execute( cmd )
263
311
264
312
#restore permissions
313
logger.info( "Restore permissions" )
265
314
file_info_dict = self.load_fileinfo_dict( snapshot_id, info_file.get_int_value( 'snapshot_version' ) )
266
315
if len( file_info_dict ) > 0:
267
316
#explore items
290
339
self._restore_path_info( item_path, file_info_dict )
291
340
292
341
def get_snapshots_list( self, sort_reverse = True ):
293
biglist = []
294
snapshots_path = self.config.get_snapshots_full_path()
295
296
try:
297
biglist = os.listdir( snapshots_path )
298
except:
299
pass
300
301
list = []
302
303
for item in biglist:
304
if len( item ) != 15:
305
continue
306
if os.path.isdir( os.path.join( snapshots_path, item ) ):
307
list.append( item )
308
342
'''Returns a list with the snapshot_ids of all snapshots in the snapshots folder'''
343
biglist = []
344
profile_id = self.config.get_current_profile()
345
snapshots_path = self.config.get_snapshots_full_path( profile_id )
346
347
try:
348
biglist = os.listdir( snapshots_path )
349
except:
350
pass
351
352
list = []
353
354
for item in biglist:
355
if len( item ) != 15 and len( item ) != 19:
356
continue
357
if os.path.isdir( os.path.join( snapshots_path, item, 'backup' ) ):
358
list.append( item )
359
360
list.sort( reverse = sort_reverse )
361
return list
362
363
def get_snapshots_and_other_list( self, sort_reverse = True ):
364
'''Returns a list with the snapshot_ids, and paths, of all snapshots in the snapshots_folder and the other_folders'''
365
366
biglist = []
367
profile_id = self.config.get_current_profile()
368
snapshots_path = self.config.get_snapshots_full_path( profile_id )
369
snapshots_other_paths = self.config.get_other_folders_paths()
370
371
try:
372
biglist = os.listdir( snapshots_path )
373
except:
374
pass
375
376
list = []
377
378
for item in biglist:
379
if len( item ) != 15 and len( item ) != 19:
380
continue
381
if os.path.isdir( os.path.join( snapshots_path, item, 'backup' ) ):
382
#a = ( item, snapshots_path )
383
list.append( item )
384
385
386
if len( snapshots_other_paths ) > 0:
387
for folder in snapshots_other_paths:
388
folderlist = []
389
try:
390
folderlist = os.listdir( folder )
391
except:
392
pass
393
394
for member in folderlist:
395
if len( member ) != 15 and len( member ) != 19:
396
continue
397
if os.path.isdir( os.path.join( folder, member, 'backup' ) ):
398
#a = ( member, folder )
399
list.append( member )
400
309
401
list.sort( reverse = sort_reverse )
310
402
return list
311
403
314
406
return
315
407
316
408
path = self.get_snapshot_path( snapshot_id )
317
cmd = "chmod -R a+rwx \"%s\"" % path
409
#cmd = "chmod -R a+rwx \"%s\"" % path
410
cmd = "find \"%s\" -type d -exec chmod u+wx {} \\;" % path #Debian patch
318
411
self._execute( cmd )
319
412
cmd = "rm -rfv \"%s\"" % path
320
413
self._execute( cmd )
321
414
415
def copy_snapshot( self, snapshot_id, new_folder ):
416
'''Copies a known snapshot to a new location'''
417
current.path = self.get_snapshot_path( snapshot_id )
418
#need to implement hardlinking to existing folder -> cp newest snapshot folder, rsync -aEAXHv --delete to this folder
419
cmd = "cp -al \"%s\"* \"%s\"" % ( current_path, new_folder )
420
logger.info( '%s is copied to folder %s' %( snapshot_id, new_folder ) )
421
self._execute( cmd )
422
322
423
def _get_last_snapshot_info( self ):
323
424
lines = ''
324
425
dict = {}
371
472
if not self.config.is_configured():
372
473
logger.warning( 'Not configured' )
373
474
self.plugin_manager.on_error( 1 ) #not configured
475
elif self.config.is_no_on_battery_enabled() and tools.on_battery():
476
logger.info( 'Deferring backup while on battery' )
477
logger.warning( 'Backup not performed' )
478
elif self.config.get_update_other_folders() == True:
479
logger.info( 'The application needs to change the backup format. Start the GUI to proceed. (As long as you do not you will not be able to make new snapshots!)' )
480
logger.warning( 'Backup not performed' )
374
481
else:
375
482
instance = applicationinstance.ApplicationInstance( self.config.get_take_snapshot_instance_file(), False )
376
483
if not instance.check():
377
484
logger.warning( 'A backup is already running' )
378
485
self.plugin_manager.on_error( 2 ) #a backup is already running
379
486
else:
487
if self.config.is_no_on_battery_enabled () and not tools.power_status_available():
488
logger.warning( 'Backups disabled on battery but power status is not available' )
489
380
490
instance.start_application()
381
491
logger.info( 'Lock' )
382
492
393
503
logger.info( 'Nothing to do' )
394
504
else:
395
505
self.plugin_manager.on_process_begins() #take snapshot process begin
396
506
logger.info( "on process begins" )
397
507
self.set_take_snapshot_message( 0, '...' )
398
399
if not self.config.can_backup():
508
profile_id = self.config.get_current_profile()
509
logger.info( "Profile_id: %s" % profile_id )
510
511
if not self.config.can_backup( profile_id ):
400
512
if self.plugin_manager.has_gui_plugins() and self.config.is_notify_enabled():
401
513
for counter in xrange( 30, 0, -1 ):
402
514
self.set_take_snapshot_message( 1,
407
519
if self.config.can_backup():
408
520
break
409
521
410
if not self.config.can_backup():
522
if not self.config.can_backup( profile_id ):
411
523
logger.warning( 'Can\'t find snapshots folder !' )
412
524
self.plugin_manager.on_error( 3 ) #Can't find snapshots directory (is it on a removable drive ?)
413
525
else:
414
526
snapshot_id = self.get_snapshot_id( now )
415
527
snapshot_path = self.get_snapshot_path( snapshot_id )
416
528
417
529
if os.path.exists( snapshot_path ):
418
530
logger.warning( "Snapshot path \"%s\" already exists" % snapshot_path )
419
531
self.plugin_manager.on_error( 4, snapshot_id ) #This snapshots already exists
572
684
573
685
new_snapshot_id = 'new_snapshot'
574
686
new_snapshot_path = self.get_snapshot_path( new_snapshot_id )
575
687
576
688
if os.path.exists( new_snapshot_path ):
577
689
#self._execute( "find \"%s\" -type d -exec chmod +w {} \;" % new_snapshot_path )
578
self._execute( "chmod -R a+rwx \"%s\"" % new_snapshot_path )
690
#self._execute( "chmod -R a+rwx \"%s\"" % new_snapshot_path )
691
self._execute( "find \"%s\" -type d -exec chmod u+wx {} \\;" % new_snapshot_path ) #Debian patch
579
692
self._execute( "rm -rf \"%s\"" % new_snapshot_path )
580
693
581
694
if os.path.exists( new_snapshot_path ):
585
698
return False
586
699
587
700
new_snapshot_path_to = self.get_snapshot_path_to( new_snapshot_id )
588
701
589
702
#create exclude patterns string
590
703
items = []
591
704
for exclude in self.config.get_exclude_patterns():
608
721
rsync_include2 = ' '.join( items2 )
609
722
610
723
#rsync prefix & suffix
611
rsync_prefix = 'rsync -aEAX '
724
rsync_prefix = 'rsync -aEAXH '
612
725
rsync_exclude_backup_directory = " --exclude=\"%s\" --exclude=\"%s\" " % ( self.config.get_snapshots_path(), self.config._LOCAL_DATA_FOLDER )
613
rsync_suffix = ' --chmod=Fa-w,D+w --whole-file --delete ' + rsync_exclude_backup_directory + rsync_include + ' ' + rsync_exclude + ' ' + rsync_include2 + ' --exclude=\"*\" / '
726
rsync_suffix = ' --chmod=Fa-w,Da-w --whole-file --delete ' + rsync_exclude_backup_directory + rsync_include + ' ' + rsync_exclude + ' ' + rsync_include2 + ' --exclude=\"*\" / '
614
727
615
728
#update dict
616
729
if not force:
620
733
self._set_last_snapshot_info( dict )
621
734
622
735
#check previous backup
736
#should only contain the personal snapshots
623
737
snapshots = self.get_snapshots_list()
624
738
prev_snapshot_id = ''
625
739
740
if len( snapshots ) == 0:
741
snapshots = self.get_snapshots_and_other_list()
742
# When there is no snapshots it takes the last snapshot from the other folders
743
# It should delete the excluded folders then
744
rsync_prefix = rsync_prefix + '--delete-excluded '
745
746
626
747
if len( snapshots ) > 0:
627
748
prev_snapshot_id = snapshots[0]
628
749
prev_snapshot_name = self.get_snapshot_display_id( prev_snapshot_id )
652
773
self.set_take_snapshot_message( 0, _('Create hard-links') )
653
774
logger.info( "Create hard-links" )
654
775
655
if force or len( ignore_folders ) == 0:
656
cmd = "cp -al \"%s\"* \"%s\"" % ( self.get_snapshot_path_to( prev_snapshot_id ), new_snapshot_path_to )
657
self._execute( cmd )
658
else:
659
for folder in include_folders:
660
prev_path = self.get_snapshot_path_to( prev_snapshot_id, folder )
661
new_path = self.get_snapshot_path_to( new_snapshot_id, folder )
662
tools.make_dirs( new_path )
663
cmd = "cp -alb \"%s\"* \"%s\"" % ( prev_path, new_path )
664
self._execute( cmd )
776
# When schedule per included folders is enabled this did not work (cp -alb iso cp -al?)
777
# This resulted in a complete rsync for the whole snapshot consuming time and space
778
# The ignored folders were copied afterwards. To solve this, the whole last snapshot is now hardlinked
779
# and rsync is called only for the folders that should be synced (without --delete-excluded).
780
#if force or len( ignore_folders ) == 0:
781
cmd = "cp -al \"%s\"* \"%s\"" % ( self.get_snapshot_path_to( prev_snapshot_id ), new_snapshot_path_to )
782
self._execute( cmd )
783
#else:
784
# for folder in include_folders:
785
# prev_path = self.get_snapshot_path_to( prev_snapshot_id, folder )
786
# new_path = self.get_snapshot_path_to( new_snapshot_id, folder )
787
# tools.make_dirs( new_path )
788
# cmd = "cp -alb \"%s\"* \"%s\"" % ( prev_path, new_path )
789
# self._execute( cmd )
665
790
else:
666
791
if not self._create_directory( new_snapshot_path_to ):
667
792
return False
668
793
669
794
#sync changed folders
670
795
logger.info( "Call rsync to take the snapshot" )
671
cmd = rsync_prefix + ' -v --delete-excluded ' + rsync_suffix + '"' + new_snapshot_path_to + '"'
796
cmd = rsync_prefix + ' -v ' + rsync_suffix + '"' + new_snapshot_path_to + '"' # do not delete the excluded, as we will miss the hardlinks with files or folders that are scheduled for a later time
672
797
self.set_take_snapshot_message( 0, _('Take snapshot') )
673
798
self._execute( cmd, self._exec_rsync_callback )
674
799
687
812
fileinfo_dict[item_path] = 1
688
813
self._save_path_info( fileinfo, item_path )
689
814
690
#copy ignored folders
691
if not force and len( prev_snapshot_id ) > 0 and len( ignore_folders ) > 0:
692
prev_fileinfo_dict = self.load_fileinfo_dict( prev_snapshot_id )
693
694
for folder in ignore_folders:
695
prev_path = self.get_snapshot_path_to( prev_snapshot_id, folder )
696
new_path = self.get_snapshot_path_to( new_snapshot_id, folder )
697
tools.make_dirs( new_path )
698
cmd = "cp -alb \"%s/\"* \"%s\"" % ( prev_path, new_path )
699
self._execute( cmd )
700
701
if len( prev_fileinfo_dict ) > 0:
702
#save permissions for all items to folder
703
item_path = '/'
704
prev_path_items = folder.strip( '/' ).split( '/' )
705
for item in items:
706
item_path = os.path.join( item_path, item )
707
if item_path not in fileinfo_dict and item_path in prev_fileinfo_dict:
708
self._save_path_info_line( fileinfo, item_path, prev_fileinfo_dict[item_path] )
709
710
#save permission for all items in folder
711
for path, dirs, files in os.walk( new_path ):
712
dirs.extend( files )
713
for item in dirs:
714
item_path = os.path.join( path, item )[ len( path_to_explore ) : ]
715
if item_path not in fileinfo_dict and item_path in prev_fileinfo_dict:
716
self._save_path_info_line( fileinfo, item_path, prev_fileinfo_dict[item_path] )
815
# We now copy on forehand, so copying afterwards is not necessary anymore
816
##copy ignored folders
817
#if not force and len( prev_snapshot_id ) > 0 and len( ignore_folders ) > 0:
818
# prev_fileinfo_dict = self.load_fileinfo_dict( prev_snapshot_id )
819
#
820
# for folder in ignore_folders:
821
# prev_path = self.get_snapshot_path_to( prev_snapshot_id, folder )
822
# new_path = self.get_snapshot_path_to( new_snapshot_id, folder )
823
# tools.make_dirs( new_path )
824
# cmd = "cp -alb \"%s/\"* \"%s\"" % ( prev_path, new_path )
825
# self._execute( cmd )
826
#
827
# if len( prev_fileinfo_dict ) > 0:
828
# #save permissions for all items to folder
829
# item_path = '/'
830
# prev_path_items = folder.strip( '/' ).split( '/' )
831
# for item in items:
832
# item_path = os.path.join( item_path, item )
833
# if item_path not in fileinfo_dict and item_path in prev_fileinfo_dict:
834
# self._save_path_info_line( fileinfo, item_path, prev_fileinfo_dict[item_path] )
835
836
# #save permission for all items in folder
837
# for path, dirs, files in os.walk( new_path ):
838
# dirs.extend( files )
839
# for item in dirs:
840
# item_path = os.path.join( path, item )[ len( path_to_explore ) : ]
841
# if item_path not in fileinfo_dict and item_path in prev_fileinfo_dict:
842
# self._save_path_info_line( fileinfo, item_path, prev_fileinfo_dict[item_path] )
717
843
718
844
fileinfo.close()
719
845
720
#create info file
846
#create info file
847
logger.info( "Create info file" )
848
machine = socket.gethostname()
849
user = os.environ['LOGNAME']
850
profile_id = self.config.get_current_profile()
851
tag = self.config.get_tag( profile_id )
721
852
info_file = configfile.ConfigFile()
722
info_file.set_int_value( 'snapshot_version', 1 )
853
info_file.set_int_value( 'snapshot_version', self.SNAPSHOT_VERSION )
854
info_file.set_str_value( 'snapshot_date', snapshot_id[0:15] )
855
info_file.set_str_value( 'snapshot_machine', machine )
856
info_file.set_str_value( 'snapshot_user', user )
857
info_file.set_int_value( 'snapshot_profile_id', profile_id )
858
info_file.set_int_value( 'snapshot_tag', tag )
723
859
info_file.save( self.get_snapshot_info_path( new_snapshot_id ) )
724
860
info_file = None
725
861
765
901
766
902
def smart_remove( self, now_full = None ):
767
903
snapshots = self.get_snapshots_list()
904
logger.info( "[smart remove] considered: %s" % snapshots )
768
905
if len( snapshots ) <= 1:
769
906
logger.info( "[smart remove] There is only one snapshots, so keep it" )
770
907
return
827
964
snapshots = self.get_snapshots_list( False )
828
965
829
966
old_backup_id = self.get_snapshot_id( self.config.get_remove_old_snapshots_date() )
830
logger.info( "Remove backups older than: %s" % old_backup_id )
967
logger.info( "Remove backups older than: %s" % old_backup_id[0:15] )
831
968
832
969
while True:
833
970
if len( snapshots ) <= 1:
Loggerhead is a web-based interface for Breezy
Version: 2.0.1