18
18
from __future__ import with_statement
20
from twisted.web2.dav.fileop import rmdir
21
from twisted.web2.dav import davxml
22
from twistedcaldav.directory.directory import DirectoryService
23
from twistedcaldav.directory.calendaruserproxy import CalendarUserProxyDatabase
20
import xattr, os, zlib, hashlib, datetime, pwd, grp, shutil, errno
21
from zlib import compress
22
from cPickle import loads as unpickle, UnpicklingError
24
from twext.web2.dav.fileop import rmdir
25
from twext.web2.dav import davxml
26
from twext.python.log import Logger
27
from twisted.python.reflect import namedClass
30
from twistedcaldav.directory.appleopendirectory import OpenDirectoryService
31
from twistedcaldav.directory.xmlfile import XMLDirectoryService
32
from twistedcaldav.directory.directory import DirectoryService, GroupMembershipCacheUpdater
33
from twistedcaldav.directory import calendaruserproxy
34
from twistedcaldav.directory.calendaruserproxyloader import XMLCalendarUserProxyLoader
24
35
from twistedcaldav.directory.resourceinfo import ResourceInfoDatabase
25
36
from twistedcaldav.mail import MailGatewayTokensDatabase
26
from twistedcaldav.log import Logger
27
37
from twistedcaldav.ical import Component
28
38
from twistedcaldav import caldavxml
39
from twistedcaldav.scheduling.cuaddress import LocalCalendarUser
40
from twistedcaldav.scheduling.scheduler import DirectScheduler
43
from twisted.application.service import Service
44
from twisted.internet import reactor
45
from twisted.internet.defer import inlineCallbacks, succeed, returnValue
47
from txdav.caldav.datastore.index_file import db_basename
49
from calendarserver.tap.util import getRootResource, FakeRequest, directoryFromConfig
29
51
from calendarserver.tools.util import getDirectory
30
import xattr, os, zlib, hashlib, datetime, pwd, grp, shutil
31
from zlib import compress
32
from cPickle import loads as unpickle, UnpicklingError
52
from calendarserver.tools.resources import migrateResources
54
from twisted.python.reflect import namedAny
56
deadPropertyXattrPrefix = namedAny(
57
"txdav.base.propertystore.xattr.PropertyStore.deadPropertyXattrPrefix"
60
INBOX_ITEMS = "inboxitems.txt"
65
return deadPropertyXattrPrefix + n
37
67
def getCalendarServerIDs(config):
39
69
# Determine uid/gid for ownership of directories we create here
441
498
log.warn("Done processing calendar homes")
443
migrateResourceInfo(config, directory, uid, gid)
500
yield migrateResourceInfo(config, directory, uid, gid)
444
501
createMailTokensDatabase(config, uid, gid)
446
503
if errorOccurred:
447
504
raise UpgradeError("Data upgrade failed, see error.log for details")
507
def normalizeCUAddrs(data, directory, cuaCache):
509
Normalize calendar user addresses to urn:uuid: form.
511
@param data: the calendar data to convert
513
@param directory: the directory service to lookup CUAs with
514
@type data: L{DirectoryService}
515
@param cuaCache: the dictionary to use as a cache across calls, which is
516
updated as a side-effect
517
@type cuaCache: C{dict}
518
@return: tuple of (converted calendar data, boolean signaling whether
519
there were any changes to the data)
521
cal = Component.fromString(data)
523
def lookupFunction(cuaddr):
525
# Return cached results, if any.
526
if cuaCache.has_key(cuaddr):
527
return cuaCache[cuaddr]
530
principal = directory.principalForCalendarUserAddress(cuaddr)
532
log.debug("Lookup of %s failed: %s" % (cuaddr, e))
535
if principal is None:
536
result = (None, None, None)
538
rec = principal.record
540
# RFC5545 syntax does not allow backslash escaping in
541
# parameter values. A double-quote is thus not allowed
542
# in a parameter value except as the start/end delimiters.
543
# Single quotes are allowed, so we convert any double-quotes
545
fullName = rec.fullName.replace('"', "'")
547
result = (fullName, rec.guid, rec.calendarUserAddresses)
550
cuaCache[cuaddr] = result
553
cal.normalizeCalendarUserAddresses(lookupFunction)
556
return newData, not newData == data
561
def upgrade_to_2(config, directory):
563
errorOccurred = False
569
oldFilename = "calendaruserproxy.sqlite"
570
newFilename = "proxies.sqlite"
572
oldDbPath = os.path.join(config.DataRoot, oldFilename)
573
newDbPath = os.path.join(config.DataRoot, newFilename)
574
if os.path.exists(oldDbPath) and not os.path.exists(newDbPath):
575
os.rename(oldDbPath, newDbPath)
577
def flattenHome(calHome):
579
log.debug("Flattening calendar home: %s" % (calHome,))
582
for cal in os.listdir(calHome):
583
calPath = os.path.join(calHome, cal)
584
if not os.path.isdir(calPath):
585
# Skip non-directories; these might have been uploaded by a
586
# random DAV client, they can't be calendar collections.
589
if cal in ("dropbox",):
591
if os.path.exists(os.path.join(calPath, db_basename)):
594
# Commented this out because it is only needed if there are calendars nested inside of regular collections.
595
# Whilst this is technically possible in early versions of the servers the main clients did not support it.
596
# Also, the v1 upgrade does not look at nested calendars for cu-address normalization.
597
# However, we do still need to "ignore" regular collections in the calendar home so what we do is rename them
598
# with a ".collection." prefix.
599
# def scanCollection(collection):
601
# for child in os.listdir(collection):
602
# childCollection = os.path.join(collection, child)
603
# if os.path.isdir(childCollection):
604
# if os.path.exists(os.path.join(childCollection, db_basename)):
605
# newPath = os.path.join(calHome, child)
606
# if os.path.exists(newPath):
607
# newPath = os.path.join(calHome, str(uuid.uuid4()))
608
# log.debug("Moving up calendar: %s" % (childCollection,))
609
# os.rename(childCollection, newPath)
611
# scanCollection(childCollection)
613
if os.path.isdir(calPath):
614
#log.debug("Regular collection scan: %s" % (calPath,))
615
#scanCollection(calPath)
616
log.warn("Regular collection hidden: %s" % (calPath,))
617
os.rename(calPath, os.path.join(calHome, ".collection." + os.path.basename(calPath)))
620
log.error("Failed to upgrade calendar home %s: %s" % (calHome, e))
621
return succeed(False)
627
Make sure calendars inside regular collections are all moved to the top level.
629
errorOccurred = False
631
log.debug("Flattening calendar homes")
633
docRoot = config.DocumentRoot
634
if os.path.exists(docRoot):
635
calRoot = os.path.join(docRoot, "calendars")
636
if os.path.exists(calRoot) and os.path.isdir(calRoot):
637
uidHomes = os.path.join(calRoot, "__uids__")
638
if os.path.isdir(uidHomes):
639
for path1 in os.listdir(uidHomes):
640
uidLevel1 = os.path.join(uidHomes, path1)
641
if not os.path.isdir(uidLevel1):
643
for path2 in os.listdir(uidLevel1):
644
uidLevel2 = os.path.join(uidLevel1, path2)
645
if not os.path.isdir(uidLevel2):
647
for home in os.listdir(uidLevel2):
648
calHome = os.path.join(uidLevel2, home)
649
if not os.path.isdir(calHome):
651
if not flattenHome(calHome):
658
# Move auto-schedule from resourceinfo sqlite to augments:
659
yield migrateAutoSchedule(config, directory)
661
errorOccurred = flattenHomes()
664
raise UpgradeError("Data upgrade failed, see error.log for details")
450
667
# The on-disk version number (which defaults to zero if .calendarserver_version
451
668
# doesn't exist), is compared with each of the numbers in the upgradeMethods
452
669
# array. If it is less than the number, the associated method is called.
454
671
upgradeMethods = [
455
672
(1, upgrade_to_1),
458
677
def upgradeData(config):
679
directory = getDirectory()
682
# Migrate locations/resources now because upgrade_to_1 depends on them
683
# being in resources.xml
684
(yield migrateFromOD(config, directory))
686
raise UpgradeError("Unable to migrate locations and resources from OD: %s" % (e,))
460
688
docRoot = config.DocumentRoot
462
690
versionFilePath = os.path.join(docRoot, ".calendarserver_version")
595
823
# Can't rename, must copy/delete
596
824
shutil.copy2(srcPath, destPath)
597
825
os.remove(srcPath)
828
DELETECHARS = ''.join(chr(i) for i in xrange(32) if i not in (10, 13))
829
def removeIllegalCharacters(data):
831
Remove all characters below ASCII 32 except NL and CR
833
Return tuple with the processed data, and a boolean indicating wether
836
beforeLen = len(data)
837
data = data.translate(None, DELETECHARS)
839
if afterLen != beforeLen:
846
def migrateFromOD(config, directory):
848
# Migrates locations and resources from OD
850
triggerFile = "trigger_resource_migration"
851
triggerPath = os.path.join(config.ServerRoot, triggerFile)
852
if os.path.exists(triggerPath):
853
os.remove(triggerPath)
855
log.warn("Migrating locations and resources")
857
userService = directory.serviceForRecordType("users")
858
resourceService = directory.serviceForRecordType("resources")
860
not isinstance(userService, OpenDirectoryService) or
861
not isinstance(resourceService, XMLDirectoryService)
863
# Configuration requires no migration
866
# Create internal copies of resources and locations based on what is
868
return migrateResources(userService, resourceService)
872
def migrateAutoSchedule(config, directory):
873
# Fetch the autoSchedule assignments from resourceinfo.sqlite and store
874
# the values in augments
875
augmentService = directory.augmentService
877
dbPath = os.path.join(config.DataRoot, ResourceInfoDatabase.dbFilename)
878
if os.path.exists(dbPath):
879
resourceInfoDatabase = ResourceInfoDatabase(config.DataRoot)
880
results = resourceInfoDatabase._db_execute(
881
"select GUID, AUTOSCHEDULE from RESOURCEINFO"
883
for guid, autoSchedule in results:
884
record = directory.recordWithGUID(guid)
885
if record is not None:
886
augmentRecord = (yield augmentService.getAugmentRecord(guid, record.recordType))
887
augmentRecord.autoSchedule = autoSchedule
888
augmentRecords.append(augmentRecord)
890
yield augmentService.addAugmentRecords(augmentRecords)
893
class UpgradeFileSystemFormatService(Service, object):
895
Upgrade filesystem from previous versions.
898
def __init__(self, config, service):
900
Initialize the service.
902
self.wrappedService = service
909
Do the upgrade. Called by C{startService}, but a different method
910
because C{startService} should return C{None}, not a L{Deferred}.
912
@return: a Deferred which fires when the upgrade is complete.
915
# Don't try to use memcached during upgrade; it's not necessarily
917
memcacheEnabled = self.config.Memcached.Pools.Default.ClientEnabled
918
self.config.Memcached.Pools.Default.ClientEnabled = False
920
yield upgradeData(self.config)
922
# Restore memcached client setting
923
self.config.Memcached.Pools.Default.ClientEnabled = memcacheEnabled
925
# see http://twistedmatrix.com/trac/ticket/4649
926
reactor.callLater(0, self.wrappedService.setServiceParent, self.parent)
929
def startService(self):
937
class PostDBImportService(Service, object):
939
Service which runs after database import but before workers are spawned
940
(except memcached will be running at this point)
942
The jobs carried out here are:
944
1. Populating the group-membership cache
945
2. Processing non-implicit inbox items
948
def __init__(self, config, store, service):
950
Initialize the service.
952
self.wrappedService = service
957
def startService(self):
962
# Load proxy assignments from XML if specified
963
if self.config.ProxyLoadFromFile:
964
proxydbClass = namedClass(self.config.ProxyDBService.type)
965
calendaruserproxy.ProxyDBService = proxydbClass(
966
**self.config.ProxyDBService.params)
967
loader = XMLCalendarUserProxyLoader(self.config.ProxyLoadFromFile)
968
yield loader.updateProxyDB()
970
# Populate the group membership cache
971
if (self.config.GroupCaching.Enabled and
972
self.config.GroupCaching.EnableUpdater):
973
proxydb = calendaruserproxy.ProxyDBService
975
proxydbClass = namedClass(self.config.ProxyDBService.type)
976
proxydb = proxydbClass(**self.config.ProxyDBService.params)
977
directory = directoryFromConfig(self.config)
979
updater = GroupMembershipCacheUpdater(proxydb,
980
directory, self.config.GroupCaching.ExpireSeconds,
981
self.config.GroupCaching.LockSeconds,
982
namespace=self.config.GroupCaching.MemcachedPool,
983
useExternalProxies=self.config.GroupCaching.UseExternalProxies)
984
yield updater.updateCache(fast=True)
986
uid, gid = getCalendarServerIDs(self.config)
987
dbPath = os.path.join(self.config.DataRoot, "proxies.sqlite")
988
if os.path.exists(dbPath):
989
os.chown(dbPath, uid, gid)
991
# Process old inbox items
992
yield self.processInboxItems()
996
def processInboxItems(self):
998
When data is migrated from a non-implicit scheduling server there can
999
be inbox items that clients have not yet processed. This method
1000
runs those inbox items through the implicit scheduling mechanism.
1003
inboxItemsList = os.path.join(self.config.DataRoot, INBOX_ITEMS)
1004
if os.path.exists(inboxItemsList):
1006
root = getRootResource(self.config, self.store)
1007
directory = root.getDirectory()
1008
principalCollection = directory.principalCollection
1011
with open(inboxItemsList) as input:
1012
for inboxItem in input:
1013
inboxItem = inboxItem.strip()
1014
inboxItems.add(inboxItem)
1017
for inboxItem in list(inboxItems):
1018
log.info("Processing inbox item: %s" % (inboxItem,))
1019
ignore, uuid, ignore, fileName = inboxItem.rsplit("/", 3)
1021
record = directory.recordWithUID(uuid)
1025
principal = principalCollection.principalForRecord(record)
1029
request = FakeRequest(root, "PUT", None)
1030
request.checkedSACL = True
1031
request.authnUser = request.authzUser = davxml.Principal(
1032
davxml.HRef.fromString("/principals/__uids__/%s/" % (uuid,))
1035
calendarHome = yield principal.calendarHome(request)
1036
if not calendarHome:
1039
inbox = yield calendarHome.getChild("inbox")
1040
if inbox and inbox.exists():
1042
inboxItemResource = yield inbox.getChild(fileName)
1043
if inboxItemResource and inboxItemResource.exists():
1045
uri = "/calendars/__uids__/%s/inbox/%s" % (uuid,
1048
request._rememberResource(inboxItemResource, uri)
1051
yield self.processInboxItem(
1061
except Exception, e:
1062
log.error("Error processing inbox item: %s (%s)"
1065
inboxItems.remove(inboxItem)
1069
# Rewrite the inbox items file in case we exit before we're
1070
# done so we'll pick up where we left off next time we start up.
1072
with open(inboxItemsList + ".tmp", "w") as output:
1073
for inboxItem in inboxItems:
1074
output.write("%s\n" % (inboxItem,))
1075
os.rename(inboxItemsList + ".tmp", inboxItemsList)
1076
log.error("Restart calendar service to reattempt processing")
1078
os.remove(inboxItemsList)
1080
reactor.callLater(0, self.wrappedService.setServiceParent, self.parent)
1084
def processInboxItem(self, root, directory, principal, request, inbox,
1085
inboxItem, uuid, uri):
1087
Run an individual inbox item through implicit scheduling and remove
1091
log.debug("Processing inbox item %s" % (inboxItem,))
1093
txn = request._newStoreTransaction
1094
txn._notifierFactory = None # Do not send push notifications
1096
ownerPrincipal = principal
1097
cua = "urn:uuid:%s" % (uuid,)
1098
owner = LocalCalendarUser(cua, ownerPrincipal,
1099
inbox, ownerPrincipal.scheduleInboxURL())
1101
calendar = yield inboxItem.iCalendar()
1102
if calendar.mainType() is not None:
1104
method = calendar.propertyValue("METHOD")
1108
if method == "REPLY":
1109
# originator is attendee sending reply
1110
originator = calendar.getAttendees()[0]
1112
# originator is the organizer
1113
originator = calendar.getOrganizer()
1115
principalCollection = directory.principalCollection
1116
originatorPrincipal = principalCollection.principalForCalendarUserAddress(originator)
1117
originator = LocalCalendarUser(originator, originatorPrincipal)
1118
recipients = (owner,)
1120
scheduler = DirectScheduler(request, inboxItem)
1121
# Process inbox item
1122
yield scheduler.doSchedulingViaPUT(originator, recipients, calendar,
1123
internal_request=False)
1125
log.warn("Removing invalid inbox item: %s" % (uri,))
1130
yield inboxItem.storeRemove(request, True, uri)
added
removed
twistedcaldav/upgrade.py
