1
# Miro - an RSS based video player application
2
# Copyright (C) 2005-2010 Participatory Culture Foundation
4
# This program is free software; you can redistribute it and/or modify
5
# it under the terms of the GNU General Public License as published by
6
# the Free Software Foundation; either version 2 of the License, or
7
# (at your option) any later version.
9
# This program is distributed in the hope that it will be useful,
10
# but WITHOUT ANY WARRANTY; without even the implied warranty of
11
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12
# GNU General Public License for more details.
14
# You should have received a copy of the GNU General Public License
15
# along with this program; if not, write to the Free Software
16
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
18
# In addition, as a special exception, the copyright holders give
19
# permission to link the code of portions of this program with the OpenSSL
22
# You must obey the GNU General Public License in all respects for all of
23
# the code used other than OpenSSL. If you modify file(s) with this
24
# exception, you may extend this exception to your version of the file(s),
25
# but you are not obligated to do so. If you do not wish to do so, delete
26
# this exception statement from your version. If you delete this exception
27
# statement from all source files in the program, then also delete it here.
29
"""Responsible for upgrading old versions of the database.
33
For really old versions before the ``schema.py`` module, see
34
``olddatabaseupgrade.py``.
37
from urlparse import urlparse
46
from miro import schema
49
from miro import config
50
from miro import dbupgradeprogress
51
from miro import prefs
53
# looks nicer as a return value
56
class DatabaseTooNewError(Exception):
57
"""Error that we raise when we see a database that is newer than
58
the version that we can update too.
62
def remove_column(cursor, table, column_names):
63
"""Remove a column from a SQLITE table. This was added for
64
upgrade88, but it's probably useful for other ones as well.
66
:param table: the table to remove the columns from
67
:param column_names: list of columns to remove
69
cursor.execute("PRAGMA table_info('%s')" % table)
71
columns_with_type = []
72
for column_info in cursor.fetchall():
73
column = column_info[1]
74
col_type = column_info[2]
75
if column in column_names:
77
columns.append(column)
79
col_type += ' PRIMARY KEY'
80
columns_with_type.append("%s %s" % (column, col_type))
82
cursor.execute("PRAGMA index_list('%s')" % table)
84
for index_info in cursor.fetchall():
86
if name in column_names:
88
cursor.execute("SELECT sql FROM sqlite_master "
89
"WHERE name=? and type='index'", (name,))
90
index_sql.append(cursor.fetchone()[0])
92
cursor.execute("ALTER TABLE %s RENAME TO old_%s" % (table, table))
93
cursor.execute("CREATE TABLE %s (%s)" %
94
(table, ', '.join(columns_with_type)))
95
cursor.execute("INSERT INTO %s SELECT %s FROM old_%s" %
96
(table, ', '.join(columns), table))
97
cursor.execute("DROP TABLE old_%s" % table)
101
def get_object_tables(cursor):
102
"""Returns a list of tables that store ``DDBObject`` subclasses.
104
cursor.execute("SELECT name FROM sqlite_master "
105
"WHERE type='table' AND name != 'dtv_variables' AND "
106
"name NOT LIKE 'sqlite%'")
107
return [row[0] for row in cursor]
109
def get_next_id(cursor):
110
"""Calculate the next id to assign to new rows.
112
This will be 1 higher than the max id for all the tables in the
116
for table in get_object_tables(cursor):
117
cursor.execute("SELECT MAX(id) from %s" % table)
118
max_id = max(max_id, cursor.fetchone()[0])
121
_upgrade_overide = {}
122
def get_upgrade_func(version):
123
if version in _upgrade_overide:
124
return _upgrade_overide[version]
126
return globals()['upgrade%d' % version]
128
def new_style_upgrade(cursor, saved_version, upgrade_to):
129
"""Upgrade a database using new-style upgrade functions.
131
This method replaces the upgrade() method. However, we still need
132
to keep around upgrade() to upgrade old databases. We switched
133
upgrade styles at version 80.
135
This method will call upgradeX for each number X between
136
saved_version and upgrade_to. cursor should be a SQLite database
137
cursor that will be passed to each upgrade function. For example,
138
if save_version is 2 and upgrade_to is 4, this method is
145
if saved_version > upgrade_to:
146
msg = ("Database was created by a newer version of Miro "
147
"(db version is %s)" % saved_version)
148
raise DatabaseTooNewError(msg)
150
dbupgradeprogress.new_style_progress(saved_version, saved_version,
152
for version in xrange(saved_version + 1, upgrade_to + 1):
154
logging.info("upgrading database to version %s", version)
155
get_upgrade_func(version)(cursor)
156
dbupgradeprogress.new_style_progress(saved_version, version,
159
def upgrade(savedObjects, saveVersion, upgradeTo=None):
160
"""Upgrade a list of SavableObjects that were saved using an old
161
version of the database schema.
163
This method will call upgradeX for each number X between
164
saveVersion and upgradeTo. For example, if saveVersion is 2 and
165
upgradeTo is 4, this method is equivelant to::
167
upgrade3(savedObjects)
168
upgrade4(savedObjects)
170
By default, upgradeTo will be the VERSION variable in schema.
174
if upgradeTo is None:
175
upgradeTo = schema.VERSION
177
if saveVersion > upgradeTo:
178
msg = ("Database was created by a newer version of Miro "
179
"(db version is %s)" % saveVersion)
180
raise DatabaseTooNewError(msg)
182
startSaveVersion = saveVersion
183
dbupgradeprogress.old_style_progress(startSaveVersion, startSaveVersion,
185
while saveVersion < upgradeTo:
187
print "upgrading database to version %s" % (saveVersion + 1)
188
upgradeFunc = get_upgrade_func(saveVersion + 1)
189
thisChanged = upgradeFunc(savedObjects)
190
if thisChanged is None or changed is None:
193
changed.update (thisChanged)
195
dbupgradeprogress.old_style_progress(startSaveVersion, saveVersion,
199
def upgrade2(objectList):
200
"""Add a dlerType variable to all RemoteDownloader objects."""
202
if o.classString == 'remote-downloader':
203
# many of our old attributes are now stored in status
204
o.savedData['status'] = {}
205
for key in ('startTime', 'endTime', 'filename', 'state',
206
'currentSize', 'totalSize', 'reasonFailed'):
207
o.savedData['status'][key] = o.savedData[key]
209
# force the download daemon to create a new downloader object.
210
o.savedData['dlid'] = 'noid'
212
def upgrade3(objectList):
213
"""Add the expireTime variable to FeedImpl objects."""
215
if o.classString == 'feed':
216
feedImpl = o.savedData['actualFeed']
217
if feedImpl is not None:
218
feedImpl.savedData['expireTime'] = None
220
def upgrade4(objectList):
221
"""Add iconCache variables to all Item objects."""
223
if o.classString in ['item', 'file-item', 'feed']:
224
o.savedData['iconCache'] = None
226
def upgrade5(objectList):
227
"""Upgrade metainfo from old BitTorrent format to BitTornado format"""
229
if o.classString == 'remote-downloader':
230
if o.savedData['status'].has_key('metainfo'):
231
o.savedData['status']['metainfo'] = None
232
o.savedData['status']['infohash'] = None
234
def upgrade6(objectList):
235
"""Add downloadedTime to items."""
237
if o.classString in ('item', 'file-item'):
238
o.savedData['downloadedTime'] = None
240
def upgrade7(objectList):
241
"""Add the initialUpdate variable to FeedImpl objects."""
243
if o.classString == 'feed':
244
feedImpl = o.savedData['actualFeed']
245
if feedImpl is not None:
246
feedImpl.savedData['initialUpdate'] = False
248
def upgrade8(objectList):
249
"""Have items point to feed_id instead of feed."""
251
if o.classString in ('item', 'file-item'):
252
o.savedData['feed_id'] = o.savedData['feed'].savedData['id']
254
def upgrade9(objectList):
255
"""Added the deleted field to file items"""
257
if o.classString == 'file-item':
258
o.savedData['deleted'] = False
260
def upgrade10(objectList):
261
"""Add a watchedTime attribute to items. Since we don't know when
262
that was, we use the downloaded time which matches with our old
268
if o.classString in ('item', 'file-item'):
269
if o.savedData['seen']:
270
o.savedData['watchedTime'] = o.savedData['downloadedTime']
272
o.savedData['watchedTime'] = None
276
def upgrade11(objectList):
277
"""We dropped the loadedThisSession field from ChannelGuide. No
278
need to change anything for this."""
281
def upgrade12(objectList):
282
from miro import filetypes
283
from datetime import datetime
286
if o.classString in ('item', 'file-item'):
287
if not o.savedData.has_key('releaseDateObj'):
289
enclosures = o.savedData['entry'].enclosures
290
for enc in enclosures:
291
if filetypes.is_video_enclosure(enc):
294
o.savedData['releaseDateObj'] = datetime(*enclosure.updated_parsed[0:7])
295
except (SystemExit, KeyboardInterrupt):
299
o.savedData['releaseDateObj'] = datetime(*o.savedData['entry'].updated_parsed[0:7])
300
except (SystemExit, KeyboardInterrupt):
303
o.savedData['releaseDateObj'] = datetime.min
307
def upgrade13(objectList):
308
"""Add an isContainerItem field. Computing this requires reading
309
through files and we need to do this check anyway in onRestore, in
310
case it has only been half done."""
313
for i in xrange(len(objectList) - 1, -1, -1):
315
if o.classString in ('item', 'file-item'):
316
if o.savedData['feed_id'] == None:
319
o.savedData['isContainerItem'] = None
320
o.savedData['parent_id'] = None
321
o.savedData['videoFilename'] = ""
325
def upgrade14(objectList):
326
"""Add default and url fields to channel guide."""
330
if o.classString == 'channel-guide':
331
o.savedData['url'] = None
335
def upgrade15(objectList):
336
"""In the unlikely event that someone has a playlist around,
337
change items to item_ids."""
340
if o.classString == 'playlist':
341
o.savedData['item_ids'] = o.savedData['items']
345
def upgrade16(objectList):
348
if o.classString == 'file-item':
349
o.savedData['shortFilename'] = None
353
def upgrade17(objectList):
354
"""Add folder_id attributes to Feed and SavedPlaylist. Add
355
item_ids attribute to PlaylistFolder.
359
if o.classString in ('feed', 'playlist'):
360
o.savedData['folder_id'] = None
362
elif o.classString == 'playlist-folder':
363
o.savedData['item_ids'] = []
367
def upgrade18(objectList):
368
"""Add shortReasonFailed to RemoteDownloader status dicts. """
371
if o.classString == 'remote-downloader':
372
o.savedData['status']['shortReasonFailed'] = \
373
o.savedData['status']['reasonFailed']
377
def upgrade19(objectList):
378
"""Add origURL to RemoteDownloaders"""
381
if o.classString == 'remote-downloader':
382
o.savedData['origURL'] = o.savedData['url']
386
def upgrade20(objectList):
387
"""Add redirectedURL to Guides"""
390
if o.classString == 'channel-guide':
391
o.savedData['redirectedURL'] = None
392
# set cachedGuideBody to None, to force us to update redirectedURL
393
o.savedData['cachedGuideBody'] = None
397
def upgrade21(objectList):
398
"""Add searchTerm to Feeds"""
401
if o.classString == 'feed':
402
o.savedData['searchTerm'] = None
406
def upgrade22(objectList):
407
"""Add userTitle to Feeds"""
410
if o.classString == 'feed':
411
o.savedData['userTitle'] = None
415
def upgrade23(objectList):
416
"""Remove container items from playlists."""
421
if o.classString in ('playlist', 'playlist-folder'):
423
elif (o.classString in ('item', 'file-item') and
424
o.savedData['isContainerItem']):
425
toFilter.add(o.savedData['id'])
427
filtered = [id for id in p.savedData['item_ids'] if id not in toFilter]
428
if len(filtered) != len(p.savedData['item_ids']):
430
p.savedData['item_ids'] = filtered
433
def upgrade24(objectList):
434
"""Upgrade metainfo back to BitTorrent format."""
436
if o.classString == 'remote-downloader':
437
if o.savedData['status'].has_key('metainfo'):
438
o.savedData['status']['metainfo'] = None
439
o.savedData['status']['infohash'] = None
441
def upgrade25(objectList):
442
"""Remove container items from playlists."""
443
from datetime import datetime
448
if o.classString == 'feed':
449
startfroms[o.savedData['id']] = o.savedData['actualFeed'].savedData['startfrom']
451
if o.classString == 'item':
452
pubDate = o.savedData['releaseDateObj']
453
feed_id = o.savedData['feed_id']
454
if feed_id is not None and startfroms.has_key(feed_id):
455
o.savedData['eligibleForAutoDownload'] = pubDate != datetime.max and pubDate >= startfroms[feed_id]
457
o.savedData['eligibleForAutoDownload'] = False
459
if o.classString == 'file-item':
460
o.savedData['eligibleForAutoDownload'] = True
464
def upgrade26(objectList):
467
if o.classString == 'feed':
468
feedImpl = o.savedData['actualFeed']
469
for field in ('autoDownloadable', 'getEverything', 'maxNew',
470
'fallBehind', 'expire', 'expireTime'):
471
o.savedData[field] = feedImpl.savedData[field]
475
def upgrade27(objectList):
476
"""We dropped the sawIntro field from ChannelGuide. No need to
477
change anything for this."""
480
def upgrade28(objectList):
481
from miro import filetypes
482
objectList.sort(key=lambda o: o.savedData['id'])
487
def get_first_video_enclosure(entry):
488
"""Find the first video enclosure in a feedparser entry.
489
Returns the enclosure, or None if no video enclosure is found.
492
enclosures = entry.enclosures
493
except (KeyError, AttributeError):
495
for enclosure in enclosures:
496
if filetypes.is_video_enclosure(enclosure):
500
for i in xrange(len(objectList) - 1, -1, -1):
502
if o.classString == 'item':
503
entry = o.savedData['entry']
504
videoEnc = get_first_video_enclosure(entry)
505
if videoEnc is not None:
506
entryURL = videoEnc.get('url')
509
title = entry.get("title")
510
feed_id = o.savedData['feed_id']
511
if title is not None or entryURL is not None:
512
if (feed_id, entryURL, title) in items:
513
removed.add(o.savedData['id'])
517
items.add((feed_id, entryURL, title))
519
for i in xrange(len(objectList) - 1, -1, -1):
521
if o.classString == 'file-item':
522
if o.savedData['parent_id'] in removed:
527
def upgrade29(objectList):
530
if o.classString == 'guide':
531
o.savedData['default'] = (o.savedData['url'] is None)
535
def upgrade30(objectList):
538
if o.classString == 'guide':
539
if o.savedData['default']:
540
o.savedData['url'] = None
544
def upgrade31(objectList):
547
if o.classString == 'remote-downloader':
548
o.savedData['status']['retryTime'] = None
549
o.savedData['status']['retryCount'] = -1
553
def upgrade32(objectList):
556
if o.classString == 'remote-downloader':
557
o.savedData['channelName'] = None
561
def upgrade33(objectList):
564
if o.classString == 'remote-downloader':
565
o.savedData['duration'] = None
569
def upgrade34(objectList):
572
if o.classString in ('item', 'file-item'):
573
o.savedData['duration'] = None
577
def upgrade35(objectList):
580
if o.classString in ('item', 'file-item'):
581
if hasattr(o.savedData,'entry'):
582
entry = o.savedData['entry']
583
if ((entry.has_key('title')
584
and type(entry.title) != types.UnicodeType)):
585
entry.title = entry.title.decode('utf-8', 'replace')
589
def upgrade36(objectList):
592
if o.classString == 'remote-downloader':
593
o.savedData['manualUpload'] = False
597
def upgrade37(objectList):
602
if o.classString == 'feed':
603
feedImpl = o.savedData['actualFeed']
604
if feedImpl.classString == 'directory-feed-impl':
605
id = o.savedData['id']
611
for i in xrange(len(objectList) - 1, -1, -1):
613
if o.classString == 'file-item' and o.savedData['feed_id'] == id:
614
removed.add(o.savedData['id'])
618
for i in xrange(len(objectList) - 1, -1, -1):
620
if o.classString == 'file-item':
621
if o.savedData['parent_id'] in removed:
626
def upgrade38(objectList):
629
if o.classString == 'remote-downloader':
631
if o.savedData['status']['channelName']:
632
o.savedData['status']['channelName'] = o.savedData['status']['channelName'].translate({ ord('/') : u'-',
636
except (SystemExit, KeyboardInterrupt):
642
def upgrade39(objectList):
646
for i in xrange(len(objectList) - 1, -1, -1):
648
if o.classString in ('item', 'file-item'):
650
if o.savedData['parent_id']:
653
o.savedData['isVideo'] = False
654
o.savedData['videoFilename'] = ""
655
o.savedData['isContainerItem'] = None
656
if o.classString == 'file-item':
657
o.savedData['offsetPath'] = None
660
def upgrade40(objectList):
663
if o.classString in ('item', 'file-item'):
664
o.savedData['resumeTime'] = 0
668
# Turns all strings in data structure to unicode, used by upgrade 41
671
from miro.feedparser import FeedParserDict
672
from types import StringType
673
if isinstance(d, FeedParserDict):
676
d[key] = unicodify(d[key])
678
# Feedparser dicts sometime return names in keys()
679
# that can't actually be used in keys. I guess the
680
# best thing to do here is ignore it -- Ben
682
elif isinstance(d, dict):
684
d[key] = unicodify(d[key])
685
elif isinstance(d, list):
686
for key in range(len(d)):
687
d[key] = unicodify(d[key])
688
elif type(d) == StringType:
689
d = d.decode('ascii','replace')
692
def upgrade41(objectList):
693
from miro.plat.utils import FilenameType
694
# This is where John Lennon's ghost sings "Binary Fields Forever"
695
if FilenameType == str:
696
binaryFields = ['filename', 'videoFilename', 'shortFilename',
697
'offsetPath', 'initialHTML', 'status', 'channelName']
698
icStrings = ['etag', 'modified', 'url']
699
icBinary = ['filename']
700
statusBinary = ['channelName', 'shortFilename', 'filename', 'metainfo']
702
binaryFields = ['initialHTML', 'status']
703
icStrings = ['etag', 'modified', 'url', 'filename']
705
statusBinary = ['metainfo']
709
o.savedData = unicodify(o.savedData)
710
for field in o.savedData:
711
if field not in binaryFields:
712
o.savedData[field] = unicodify(o.savedData[field])
714
# These get skipped because they're a level lower
715
if field == 'actualFeed':
716
o.savedData[field].__dict__ = unicodify(o.savedData[field].__dict__)
717
elif (field == 'iconCache' and
718
o.savedData['iconCache'] is not None):
719
for icfield in icStrings:
720
o.savedData['iconCache'].savedData[icfield] = unicodify(o.savedData['iconCache'].savedData[icfield])
721
for icfield in icBinary:
722
if (type(o.savedData['iconCache'].savedData[icfield]) == unicode):
723
o.savedData['iconCache'].savedData[icfield] = o.savedData['iconCache'].savedData[icfield].encode('ascii','replace')
726
if field == 'status':
727
for subfield in o.savedData['status']:
728
if (type(o.savedData[field][subfield]) == unicode
729
and subfield in statusBinary):
730
o.savedData[field][subfield] = o.savedData[field][subfield].encode('ascii',
732
elif (type(o.savedData[field][subfield]) == str
733
and subfield not in statusBinary):
734
o.savedData[field][subfield] = o.savedData[field][subfield].decode('ascii',
736
elif type(o.savedData[field]) == unicode:
737
o.savedData[field] = o.savedData[field].encode('ascii', 'replace')
738
if o.classString == 'channel-guide':
739
del o.savedData['cachedGuideBody']
743
def upgrade42(objectList):
746
if o.classString in ('item', 'file-item'):
747
o.savedData['screenshot'] = None
751
def upgrade43(objectList):
755
for i in xrange(len(objectList) - 1, -1, -1):
757
if o.classString == 'feed':
758
feedImpl = o.savedData['actualFeed']
759
if feedImpl.classString == 'manual-feed-impl':
760
id = o.savedData['id']
763
for i in xrange(len(objectList) - 1, -1, -1):
765
if ((o.classString == 'file-item'
766
and o.savedData['feed_id'] == id
767
and o.savedData['deleted'] == True)):
768
removed.add(o.savedData['id'])
772
for i in xrange(len(objectList) - 1, -1, -1):
774
if o.classString == 'file-item':
775
if o.savedData['parent_id'] in removed:
780
def upgrade44(objectList):
783
if (('iconCache' in o.savedData
784
and o.savedData['iconCache'] is not None)):
785
iconCache = o.savedData['iconCache']
786
iconCache.savedData['resized_filenames'] = {}
790
def upgrade45(objectList):
791
"""Dropped the ChannelGuide.redirected URL attribute. Just need
792
to bump the db version number."""
795
def upgrade46(objectList):
796
"""fastResumeData should be str, not unicode."""
799
if o.classString == 'remote-downloader':
801
if type (o.savedData['status']['fastResumeData']) == unicode:
802
o.savedData['status']['fastResumeData'] = o.savedData['status']['fastResumeData'].encode('ascii','replace')
804
except (SystemExit, KeyboardInterrupt):
810
def upgrade47(objectList):
811
"""Parsed item entries must be unicode"""
814
if o.classString == 'item':
815
o.savedData['entry'] = unicodify(o.savedData['entry'])
819
def upgrade48(objectList):
824
if o.classString == 'feed':
825
feedImpl = o.savedData['actualFeed']
826
if feedImpl.classString == 'directory-watch-feed-impl':
827
ids.add (o.savedData['id'])
832
for i in xrange(len(objectList) - 1, -1, -1):
834
if o.classString == 'file-item' and o.savedData['feed_id'] in ids:
835
removed.add(o.savedData['id'])
839
for i in xrange(len(objectList) - 1, -1, -1):
841
if o.classString == 'file-item':
842
if o.savedData['parent_id'] in removed:
847
upgrade49 = upgrade42
849
def upgrade50(objectList):
850
"""Parsed item entries must be unicode"""
853
if o.classString in ('item', 'file-item'):
854
if ((o.savedData['videoFilename']
855
and o.savedData['videoFilename'][0] == '\\')):
856
o.savedData['videoFilename'] = o.savedData['videoFilename'][1:]
860
def upgrade51(objectList):
861
"""Added title field to channel guides"""
864
if o.classString in ('channel-guide'):
865
o.savedData['title'] = None
869
def upgrade52(objectList):
870
from miro import filetypes
877
"""Find the first video enclosure in a feedparser entry.
878
Returns the enclosure, or None if no video enclosure is found.
880
entry = o.savedData['entry']
883
enclosures = entry.enclosures
884
except (KeyError, AttributeError):
887
for enclosure in enclosures:
888
if filetypes.is_video_enclosure(enclosure):
895
id = entry.get('guid', id)
896
title = entry.get('title')
897
return (url, id, title)
900
if o.classString == 'feed':
901
feedImpl = o.savedData['actualFeed']
902
if feedImpl.classString == 'search-feed-impl':
903
search_id = o.savedData['id']
904
elif feedImpl.classString == 'search-downloads-feed-impl':
905
downloads_id = o.savedData['id']
908
items_by_titleURL = {}
911
if o.classString == 'item':
912
if o.savedData['feed_id'] == search_id:
913
(url, id, title) = getVideoInfo(o)
915
items_by_idURL[(id, url)] = o
917
items_by_titleURL[(title, url)] = o
918
if downloads_id != 0:
919
for i in xrange(len(objectList) - 1, -1, -1):
921
if o.classString == 'item':
922
if o.savedData['feed_id'] == downloads_id:
924
(url, id, title) = getVideoInfo(o)
926
if items_by_idURL.has_key((id, url)):
929
items_by_idURL[(id, url)] = o
931
if items_by_titleURL.has_key((title, url)):
934
items_by_titleURL[(title, url)] = o
936
removed.add(o.savedData['id'])
940
for i in xrange(len(objectList) - 1, -1, -1):
942
if o.classString == 'file-item':
943
if o.savedData['parent_id'] in removed:
948
def upgrade53(objectList):
949
"""Added favicon and icon cache field to channel guides"""
952
if o.classString in ('channel-guide'):
953
o.savedData['favicon'] = None
954
o.savedData['iconCache'] = None
955
o.savedData['updated_url'] = o.savedData['url']
959
def upgrade54(objectList):
961
if config.get(prefs.APP_PLATFORM) != "windows-xul":
964
if o.classString in ('item', 'file-item'):
965
o.savedData['screenshot'] = None
966
o.savedData['duration'] = None
970
def upgrade55(objectList):
971
"""Add resized_screenshots attribute. """
974
if o.classString in ('item', 'file-item'):
975
o.savedData['resized_screenshots'] = {}
979
def upgrade56(objectList):
980
"""Added firstTime field to channel guides"""
983
if o.classString in ('channel-guide'):
984
o.savedData['firstTime'] = False
988
def upgrade57(objectList):
989
"""Added ThemeHistory"""
994
def upgrade58(objectList):
995
"""clear fastResumeData for libtorrent"""
998
if o.classString == 'remote-downloader':
1000
o.savedData['status']['fastResumeData'] = None
1002
except (SystemExit, KeyboardInterrupt):
1008
def upgrade59(objectList):
1010
We changed ThemeHistory to allow None in the pastTheme list.
1011
Since we're upgrading, we can assume that the default channels
1012
have been added, so we'll add None to that list manually. We also
1013
require a URL for channel guides. If it's None, eplace it with
1014
https://www.miroguide.com/.
1017
for o in objectList:
1018
if o.classString == 'channel-guide' and o.savedData['url'] is None:
1019
o.savedData['url'] = u'https://www.miroguide.com/'
1021
elif o.classString == 'theme-history':
1022
if None not in o.savedData['pastThemes']:
1023
o.savedData['pastThemes'].append(None)
1027
def upgrade60(objectList):
1028
"""search feed impl is now a subclass of rss multi, so add the
1031
for o in objectList:
1032
if o.classString == 'feed':
1033
feedImpl = o.savedData['actualFeed']
1034
if feedImpl.classString == 'search-feed-impl':
1035
feedImpl.savedData['etag'] = {}
1036
feedImpl.savedData['modified'] = {}
1040
def upgrade61(objectList):
1041
"""Add resized_screenshots attribute. """
1043
for o in objectList:
1044
if o.classString in ('item', 'file-item'):
1045
o.savedData['channelTitle'] = None
1049
def upgrade62(objectList):
1050
"""Adding baseTitle to feedimpl."""
1052
for o in objectList:
1053
if o.classString == 'feed':
1054
o.savedData['baseTitle'] = None
1058
upgrade63 = upgrade37
1060
def upgrade64(objectList):
1062
for o in objectList:
1063
if o.classString == 'channel-guide':
1064
if o.savedData['url'] == config.get(prefs.CHANNEL_GUIDE_URL):
1065
allowedURLs = unicode(
1066
config.get(prefs.CHANNEL_GUIDE_ALLOWED_URLS)).split()
1067
allowedURLs.append(config.get(
1068
prefs.CHANNEL_GUIDE_FIRST_TIME_URL))
1069
o.savedData['allowedURLs'] = allowedURLs
1071
o.savedData['allowedURLs'] = []
1075
def upgrade65(objectList):
1077
for o in objectList:
1078
if o.classString == 'feed':
1079
o.savedData['maxOldItems'] = 30
1083
def upgrade66(objectList):
1085
for o in objectList:
1086
if o.classString in ('item', 'file-item'):
1087
o.savedData['title'] = u""
1091
def upgrade67(objectList):
1092
"""Add userTitle to Guides"""
1095
for o in objectList:
1096
if o.classString == 'channel-guide':
1097
o.savedData['userTitle'] = None
1101
def upgrade68(objectList):
1103
Add the 'feed section' variable
1106
for o in objectList:
1107
if o.classString in ('feed', 'channel-folder'):
1108
o.savedData['section'] = u'video'
1112
def upgrade69(objectList):
1114
Added the WidgetsFrontendState
1118
def upgrade70(objectList):
1119
"""Added for the query item in the RSSMultiFeedImpl and
1122
for o in objectList:
1123
if o.classString == 'feed':
1124
feedImpl = o.savedData['actualFeed']
1125
if feedImpl.classString in ('search-feed-impl',
1126
'rss-multi-feed-impl'):
1127
feedImpl.savedData['query'] = u""
1134
def upgrade71(objectList):
1136
Add the downloader_id attribute
1138
# So this is a crazy upgrade, because we need to use a ton of
1139
# functions. Rather than import a module, all of these were
1140
# copied from the source code from r8953 (2009-01-17). Some
1141
# slight changes were made, mostly to drop some error checking.
1143
def fix_file_urls(url):
1144
"""Fix file urls that start with file:// instead of file:///.
1145
Note: this breaks for file urls that include a hostname, but
1146
we never use those and it's not so clear what that would mean
1147
anyway--file urls is an ad-hoc spec as I can tell.
1149
if url.startswith('file://'):
1150
if not url.startswith('file:///'):
1151
url = 'file:///%s' % url[len('file://'):]
1152
url = url.replace('\\', '/')
1155
def default_port(scheme):
1156
if scheme == 'https':
1158
elif scheme == 'http':
1160
elif scheme == 'rtsp':
1162
elif scheme == 'file':
1166
def parse_url(url, split_path=False):
1167
url = fix_file_urls(url)
1168
(scheme, host, path, params, query, fragment) = util.unicodify(list(urlparse(url)))
1169
# Filter invalid URLs with duplicated ports
1170
# (http://foo.bar:123:123/baz) which seem to be part of #441.
1171
if host.count(':') > 1:
1172
host = host[0:host.rfind(':')]
1174
if scheme == '' and util.chatter:
1175
logging.warn("%r has no scheme" % url)
1178
host, port = host.split(':')
1181
except (SystemExit, KeyboardInterrupt):
1184
logging.warn("invalid port for %r" % url)
1185
port = default_port(scheme)
1187
port = default_port(scheme)
1190
scheme = scheme.lower()
1192
path = path.replace('|', ':')
1193
# Windows drive names are often specified as "C|\foo\bar"
1195
if path == '' or not path.startswith('/'):
1197
elif re.match(r'/[a-zA-Z]:', path):
1198
# Fix "/C:/foo" paths
1202
return scheme, host, port, fullPath, params, query
1205
fullPath += ';%s' % params
1207
fullPath += '?%s' % query
1208
return scheme, host, port, fullPath
1210
UNSUPPORTED_MIMETYPES = ("video/3gpp", "video/vnd.rn-realvideo",
1212
VIDEO_EXTENSIONS = ['.mov', '.wmv', '.mp4', '.m4v', '.ogg', '.ogv',
1213
'.anx', '.mpg', '.avi', '.flv', '.mpeg',
1214
'.divx', '.xvid', '.rmvb', '.mkv', '.m2v', '.ogm']
1215
AUDIO_EXTENSIONS = ['.mp3', '.m4a', '.wma', '.mka']
1216
FEED_EXTENSIONS = ['.xml', '.rss', '.atom']
1217
def is_video_enclosure(enclosure):
1219
Pass an enclosure dictionary to this method and it will return
1220
a boolean saying if the enclosure is a video or not.
1222
return (_has_video_type(enclosure) or
1223
_has_video_extension(enclosure, 'url') or
1224
_has_video_extension(enclosure, 'href'))
1226
def _has_video_type(enclosure):
1227
return ('type' in enclosure and
1228
(enclosure['type'].startswith(u'video/') or
1229
enclosure['type'].startswith(u'audio/') or
1230
enclosure['type'] == u"application/ogg" or
1231
enclosure['type'] == u"application/x-annodex" or
1232
enclosure['type'] == u"application/x-bittorrent" or
1233
enclosure['type'] == u"application/x-shockwave-flash") and
1234
(enclosure['type'] not in UNSUPPORTED_MIMETYPES))
1236
def is_allowed_filename(filename):
1238
Pass a filename to this method and it will return a boolean
1239
saying if the filename represents video, audio or torrent.
1241
return (is_video_filename(filename) or is_audio_filename(filename)
1242
or is_torrent_filename(filename))
1244
def is_video_filename(filename):
1246
Pass a filename to this method and it will return a boolean
1247
saying if the filename represents a video file.
1249
filename = filename.lower()
1250
for ext in VIDEO_EXTENSIONS:
1251
if filename.endswith(ext):
1255
def is_audio_filename(filename):
1257
Pass a filename to this method and it will return a boolean
1258
saying if the filename represents an audio file.
1260
filename = filename.lower()
1261
for ext in AUDIO_EXTENSIONS:
1262
if filename.endswith(ext):
1266
def is_torrent_filename(filename):
1268
Pass a filename to this method and it will return a boolean
1269
saying if the filename represents a torrent file.
1271
filename = filename.lower()
1272
return filename.endswith('.torrent')
1274
def _has_video_extension(enclosure, key):
1275
if key in enclosure:
1276
elems = parse_url(enclosure[key], split_path=True)
1277
return is_allowed_filename(elems[3])
1279
def get_first_video_enclosure(entry):
1281
Find the first "best" video enclosure in a feedparser entry.
1282
Returns the enclosure, or None if no video enclosure is found.
1285
enclosures = entry.enclosures
1286
except (KeyError, AttributeError):
1289
enclosures = [e for e in enclosures if is_video_enclosure(e)]
1290
if len(enclosures) == 0:
1293
enclosures.sort(cmp_enclosures)
1294
return enclosures[0]
1296
def _get_enclosure_size(enclosure):
1297
if 'filesize' in enclosure and enclosure['filesize'].isdigit():
1298
return int(enclosure['filesize'])
1302
def _get_enclosure_bitrate(enclosure):
1303
if 'bitrate' in enclosure and enclosure['bitrate'].isdigit():
1304
return int(enclosure['bitrate'])
1308
def cmp_enclosures(enclosure1, enclosure2):
1310
Returns -1 if enclosure1 is preferred, 1 if enclosure2 is
1311
preferred, and zero if there is no preference between the two
1314
# media:content enclosures have an isDefault which we should
1315
# pick since it's the preference of the feed
1316
if enclosure1.get("isDefault"):
1318
if enclosure2.get("isDefault"):
1321
# let's try sorting by preference
1322
enclosure1_index = _get_enclosure_index(enclosure1)
1323
enclosure2_index = _get_enclosure_index(enclosure2)
1324
if enclosure1_index < enclosure2_index:
1326
elif enclosure2_index < enclosure1_index:
1329
# next, let's try sorting by bitrate..
1330
enclosure1_bitrate = _get_enclosure_bitrate(enclosure1)
1331
enclosure2_bitrate = _get_enclosure_bitrate(enclosure2)
1332
if enclosure1_bitrate > enclosure2_bitrate:
1334
elif enclosure2_bitrate > enclosure1_bitrate:
1337
# next, let's try sorting by filesize..
1338
enclosure1_size = _get_enclosure_size(enclosure1)
1339
enclosure2_size = _get_enclosure_size(enclosure2)
1340
if enclosure1_size > enclosure2_size:
1342
elif enclosure2_size > enclosure1_size:
1345
# at this point they're the same for all we care
1348
def _get_enclosure_index(enclosure):
1350
return PREFERRED_TYPES.index(enclosure.get('type'))
1355
'application/x-bittorrent',
1356
'application/ogg', 'video/ogg', 'audio/ogg',
1357
'video/mp4', 'video/quicktime', 'video/mpeg',
1358
'video/x-xvid', 'video/x-divx', 'video/x-wmv',
1359
'video/x-msmpeg', 'video/x-flv']
1362
def quote_unicode_url(url):
1363
"""Quote international characters contained in a URL according
1364
to w3c, see: <http://www.w3.org/International/O-URL-code.html>
1367
for c in url.encode('utf8'):
1369
quotedChars.append(urllib.quote(c))
1371
quotedChars.append(c)
1372
return u''.join(quotedChars)
1374
# Now that that's all set, on to the actual upgrade code.
1377
url_to_downloader_id = {}
1379
for o in objectList:
1380
if o.classString == 'remote-downloader':
1381
url_to_downloader_id[o.savedData['origURL']] = o.savedData['id']
1383
for o in objectList:
1384
if o.classString in ('item', 'file-item'):
1385
entry = o.savedData['entry']
1386
videoEnclosure = get_first_video_enclosure(entry)
1387
if videoEnclosure is not None and 'url' in videoEnclosure:
1388
url = quote_unicode_url(videoEnclosure['url'].replace('+', '%20'))
1391
downloader_id = url_to_downloader_id.get(url)
1392
if downloader_id is None and hasattr(entry, 'enclosures'):
1393
# we didn't get a downloader id using
1394
# get_first_video_enclosure(), so try other enclosures.
1395
# We changed the way that function worked between
1397
for other_enclosure in entry.enclosures:
1398
if 'url' in other_enclosure:
1399
url = quote_unicode_url(other_enclosure['url'].replace('+', '%20'))
1400
downloader_id = url_to_downloader_id.get(url)
1401
if downloader_id is not None:
1403
o.savedData['downloader_id'] = downloader_id
1408
def upgrade72(objectList):
1409
"""We upgraded the database wrong in upgrade64, inadvertently
1410
adding a str to the allowedURLs list when it should be unicode.
1411
This converts that final str to unicode before the database sanity
1415
for o in objectList:
1416
if o.classString == 'channel-guide':
1417
if o.savedData['allowedURLs'] and isinstance(
1418
o.savedData['allowedURLs'][-1], str):
1419
o.savedData['allowedURLs'][-1] = unicode(
1420
o.savedData['allowedURLs'][-1])
1424
def upgrade73(objectList):
1425
"""We dropped the resized_filename attribute for icon cache
1429
def upgrade74(objectList):
1430
"""We dropped the resized_screenshots attribute for Item
1434
def upgrade75(objectList):
1435
"""Drop the entry attribute for items, replace it with a bunch
1436
individual attributes.
1438
from datetime import datetime
1440
def fix_file_urls(url):
1441
"""Fix file urls that start with file:// instead of file:///.
1442
Note: this breaks for file urls that include a hostname, but
1443
we never use those and it's not so clear what that would mean
1444
anyway--file urls is an ad-hoc spec as I can tell.
1446
if url.startswith('file://'):
1447
if not url.startswith('file:///'):
1448
url = 'file:///%s' % url[len('file://'):]
1449
url = url.replace('\\', '/')
1452
def default_port(scheme):
1453
if scheme == 'https':
1455
elif scheme == 'http':
1457
elif scheme == 'rtsp':
1459
elif scheme == 'file':
1463
def parse_url(url, split_path=False):
1464
url = fix_file_urls(url)
1465
(scheme, host, path, params, query, fragment) = util.unicodify(list(urlparse(url)))
1466
# Filter invalid URLs with duplicated ports
1467
# (http://foo.bar:123:123/baz) which seem to be part of #441.
1468
if host.count(':') > 1:
1469
host = host[0:host.rfind(':')]
1471
if scheme == '' and util.chatter:
1472
logging.warn("%r has no scheme" % url)
1475
host, port = host.split(':')
1478
except (SystemExit, KeyboardInterrupt):
1481
logging.warn("invalid port for %r" % url)
1482
port = default_port(scheme)
1484
port = default_port(scheme)
1487
scheme = scheme.lower()
1489
path = path.replace('|', ':')
1490
# Windows drive names are often specified as "C|\foo\bar"
1491
if path == '' or not path.startswith('/'):
1493
elif re.match(r'/[a-zA-Z]:', path):
1494
# Fix "/C:/foo" paths
1498
return scheme, host, port, fullPath, params, query
1501
fullPath += ';%s' % params
1503
fullPath += '?%s' % query
1504
return scheme, host, port, fullPath
1506
UNSUPPORTED_MIMETYPES = ("video/3gpp", "video/vnd.rn-realvideo",
1508
VIDEO_EXTENSIONS = ['.mov', '.wmv', '.mp4', '.m4v', '.ogg', '.ogv',
1509
'.anx', '.mpg', '.avi', '.flv', '.mpeg', '.divx',
1510
'.xvid', '.rmvb', '.mkv', '.m2v', '.ogm']
1511
AUDIO_EXTENSIONS = ['.mp3', '.m4a', '.wma', '.mka']
1512
FEED_EXTENSIONS = ['.xml', '.rss', '.atom']
1513
def is_video_enclosure(enclosure):
1514
"""Pass an enclosure dictionary to this method and it will
1515
return a boolean saying if the enclosure is a video or not.
1517
return (_has_video_type(enclosure) or
1518
_has_video_extension(enclosure, 'url') or
1519
_has_video_extension(enclosure, 'href'))
1521
def _has_video_type(enclosure):
1522
return ('type' in enclosure and
1523
(enclosure['type'].startswith(u'video/') or
1524
enclosure['type'].startswith(u'audio/') or
1525
enclosure['type'] == u"application/ogg" or
1526
enclosure['type'] == u"application/x-annodex" or
1527
enclosure['type'] == u"application/x-bittorrent" or
1528
enclosure['type'] == u"application/x-shockwave-flash") and
1529
(enclosure['type'] not in UNSUPPORTED_MIMETYPES))
1531
def is_allowed_filename(filename):
1532
"""Pass a filename to this method and it will return a boolean
1533
saying if the filename represents video, audio or torrent.
1535
return (is_video_filename(filename)
1536
or is_audio_filename(filename)
1537
or is_torrent_filename(filename))
1539
def is_video_filename(filename):
1540
"""Pass a filename to this method and it will return a boolean
1541
saying if the filename represents a video file.
1543
filename = filename.lower()
1544
for ext in VIDEO_EXTENSIONS:
1545
if filename.endswith(ext):
1549
def is_audio_filename(filename):
1550
"""Pass a filename to this method and it will return a boolean
1551
saying if the filename represents an audio file.
1553
filename = filename.lower()
1554
for ext in AUDIO_EXTENSIONS:
1555
if filename.endswith(ext):
1559
def is_torrent_filename(filename):
1560
"""Pass a filename to this method and it will return a boolean
1561
saying if the filename represents a torrent file.
1563
filename = filename.lower()
1564
return filename.endswith('.torrent')
1566
def _has_video_extension(enclosure, key):
1567
if key in enclosure:
1568
elems = parse_url(enclosure[key], split_path=True)
1569
return is_allowed_filename(elems[3])
1572
def get_first_video_enclosure(entry):
1573
"""Find the first "best" video enclosure in a feedparser
1574
entry. Returns the enclosure, or None if no video enclosure
1578
enclosures = entry.enclosures
1579
except (KeyError, AttributeError):
1582
enclosures = [e for e in enclosures if is_video_enclosure(e)]
1583
if len(enclosures) == 0:
1586
enclosures.sort(cmp_enclosures)
1587
return enclosures[0]
1589
def _get_enclosure_size(enclosure):
1590
if 'filesize' in enclosure and enclosure['filesize'].isdigit():
1591
return int(enclosure['filesize'])
1595
def _get_enclosure_bitrate(enclosure):
1596
if 'bitrate' in enclosure and enclosure['bitrate'].isdigit():
1597
return int(enclosure['bitrate'])
1601
def cmp_enclosures(enclosure1, enclosure2):
1602
"""Returns -1 if enclosure1 is preferred, 1 if enclosure2 is
1603
preferred, and zero if there is no preference between the two
1606
# media:content enclosures have an isDefault which we should
1607
# pick since it's the preference of the feed
1608
if enclosure1.get("isDefault"):
1610
if enclosure2.get("isDefault"):
1613
# let's try sorting by preference
1614
enclosure1_index = _get_enclosure_index(enclosure1)
1615
enclosure2_index = _get_enclosure_index(enclosure2)
1616
if enclosure1_index < enclosure2_index:
1618
elif enclosure2_index < enclosure1_index:
1621
# next, let's try sorting by bitrate..
1622
enclosure1_bitrate = _get_enclosure_bitrate(enclosure1)
1623
enclosure2_bitrate = _get_enclosure_bitrate(enclosure2)
1624
if enclosure1_bitrate > enclosure2_bitrate:
1626
elif enclosure2_bitrate > enclosure1_bitrate:
1629
# next, let's try sorting by filesize..
1630
enclosure1_size = _get_enclosure_size(enclosure1)
1631
enclosure2_size = _get_enclosure_size(enclosure2)
1632
if enclosure1_size > enclosure2_size:
1634
elif enclosure2_size > enclosure1_size:
1637
# at this point they're the same for all we care
1640
def _get_enclosure_index(enclosure):
1642
return PREFERRED_TYPES.index(enclosure.get('type'))
1647
'application/x-bittorrent',
1648
'application/ogg', 'video/ogg', 'audio/ogg',
1649
'video/mp4', 'video/quicktime', 'video/mpeg',
1650
'video/x-xvid', 'video/x-divx', 'video/x-wmv',
1651
'video/x-msmpeg', 'video/x-flv']
1654
def quote_unicode_url(url):
1655
"""Quote international characters contained in a URL according
1656
to w3c, see: <http://www.w3.org/International/O-URL-code.html>
1659
for c in url.encode('utf8'):
1661
quotedChars.append(urllib.quote(c))
1663
quotedChars.append(c)
1664
return u''.join(quotedChars)
1666
KNOWN_MIME_TYPES = (u'audio', u'video')
1667
KNOWN_MIME_SUBTYPES = (u'mov', u'wmv', u'mp4', u'mp3', u'mpg', u'mpeg',
1668
u'avi', u'x-flv', u'x-msvideo', u'm4v', u'mkv',
1670
MIME_SUBSITUTIONS = {
1671
u'QUICKTIME': u'MOV',
1674
def entity_replace(text):
1686
] # FIXME: have a more general, charset-aware way to do this.
1687
for src, dest in replacements:
1688
text = text.replace(src, dest)
1691
class FeedParserValues(object):
1692
"""Helper class to get values from feedparser entries
1694
FeedParserValues objects inspect the FeedParserDict for the
1695
entry attribute for various attributes using in Item
1696
(entry_title, rss_id, url, etc...).
1698
def __init__(self, entry):
1700
self.normalized_entry = normalize_feedparser_dict(entry)
1701
self.first_video_enclosure = get_first_video_enclosure(entry)
1704
'license': entry.get("license"),
1705
'rss_id': entry.get('id'),
1706
'entry_title': self._calc_title(),
1707
'thumbnail_url': self._calc_thumbnail_url(),
1708
'raw_descrption': self._calc_raw_description(),
1709
'link': self._calc_link(),
1710
'payment_link': self._calc_payment_link(),
1711
'comments_link': self._calc_comments_link(),
1712
'url': self._calc_url(),
1713
'enclosure_size': self._calc_enclosure_size(),
1714
'enclosure_type': self._calc_enclosure_type(),
1715
'enclosure_format': self._calc_enclosure_format(),
1716
'releaseDateObj': self._calc_release_date(),
1719
def update_item(self, item):
1720
for key, value in self.data.items():
1721
setattr(item, key, value)
1722
item.feedparser_output = self.normalized_entry
1724
def compare_to_item(self, item):
1725
for key, value in self.data.items():
1726
if getattr(item, key) != value:
1730
def compare_to_item_enclosures(self, item):
1731
compare_keys = ('url', 'enclosure_size', 'enclosure_type',
1733
for key in compare_keys:
1734
if getattr(item, key) != self.data[key]:
1738
def _calc_title(self):
1739
if hasattr(self.entry, "title"):
1740
# The title attribute shouldn't use entities, but some in the
1741
# wild do (#11413). In that case, try to fix them.
1742
return entity_replace(self.entry.title)
1744
if ((self.first_video_enclosure
1745
and 'url' in self.first_video_enclosure)):
1746
return self.first_video_enclosure['url'].decode("ascii",
1750
def _calc_thumbnail_url(self):
1751
"""Returns a link to the thumbnail of the video."""
1752
# Try to get the thumbnail specific to the video enclosure
1753
if self.first_video_enclosure is not None:
1754
url = self._get_element_thumbnail(self.first_video_enclosure)
1758
# Try to get any enclosure thumbnail
1759
if hasattr(self.entry, "enclosures"):
1760
for enclosure in self.entry.enclosures:
1761
url = self._get_element_thumbnail(enclosure)
1765
# Try to get the thumbnail for our entry
1766
return self._get_element_thumbnail(self.entry)
1768
def _get_element_thumbnail(self, element):
1770
thumb = element["thumbnail"]
1773
if isinstance(thumb, str):
1775
elif isinstance(thumb, unicode):
1776
return thumb.decode('ascii', 'replace')
1778
return thumb["url"].decode('ascii', 'replace')
1779
except (KeyError, AttributeError):
1782
def _calc_raw_description(self):
1785
if hasattr(self.first_video_enclosure, "text"):
1786
rv = self.first_video_enclosure["text"]
1787
elif hasattr(self.entry, "description"):
1788
rv = self.entry.description
1790
logging.exception("_calc_raw_description threw exception:")
1796
def _calc_link(self):
1797
if hasattr(self.entry, "link"):
1798
link = self.entry.link
1799
if isinstance(link, dict):
1804
if isinstance(link, unicode):
1807
return link.decode('ascii', 'replace')
1808
except UnicodeDecodeError:
1809
return link.decode('ascii', 'ignore')
1812
def _calc_payment_link(self):
1814
return self.first_video_enclosure.payment_url.decode('ascii',
1818
return self.entry.payment_url.decode('ascii','replace')
1822
def _calc_comments_link(self):
1823
return self.entry.get('comments', u"")
1825
def _calc_url(self):
1826
if ((self.first_video_enclosure is not None
1827
and 'url' in self.first_video_enclosure)):
1828
url = self.first_video_enclosure['url'].replace('+', '%20')
1829
return quote_unicode_url(url)
1833
def _calc_enclosure_size(self):
1834
enc = self.first_video_enclosure
1835
if enc is not None and "torrent" not in enc.get("type", ""):
1837
return int(enc['length'])
1838
except (KeyError, ValueError):
1841
def _calc_enclosure_type(self):
1842
if ((self.first_video_enclosure
1843
and self.first_video_enclosure.has_key('type'))):
1844
return self.first_video_enclosure['type']
1848
def _calc_enclosure_format(self):
1849
enclosure = self.first_video_enclosure
1852
extension = enclosure['url'].split('.')[-1]
1853
extension = extension.lower().encode('ascii', 'replace')
1854
except (SystemExit, KeyboardInterrupt):
1858
# Hack for mp3s, "mpeg audio" isn't clear enough
1859
if extension.lower() == u'mp3':
1861
if enclosure.get('type'):
1862
enc = enclosure['type'].decode('ascii', 'replace')
1864
mtype, subtype = enc.split('/', 1)
1865
mtype = mtype.lower()
1866
if mtype in KNOWN_MIME_TYPES:
1867
format = subtype.split(';')[0].upper()
1868
if mtype == u'audio':
1870
if format.startswith(u'X-'):
1872
return u'.%s' % MIME_SUBSITUTIONS.get(format, format).lower()
1874
if extension in KNOWN_MIME_SUBTYPES:
1875
return u'.%s' % extension
1878
def _calc_release_date(self):
1880
return datetime(*self.first_video_enclosure.updated_parsed[0:7])
1881
except (SystemExit, KeyboardInterrupt):
1885
return datetime(*self.entry.updated_parsed[0:7])
1886
except (SystemExit, KeyboardInterrupt):
1891
from datetime import datetime
1892
from time import struct_time
1893
from types import NoneType
1896
from miro import feedparser
1897
# normally we shouldn't import other modules inside an upgrade
1898
# function. However, it should be semi-safe to import feedparser,
1899
# because it would have already been imported when unpickling
1900
# FeedParserDict objects.
1902
# values from feedparser dicts that don't have to convert in
1903
# normalize_feedparser_dict()
1904
_simple_feedparser_values = (int, long, str, unicode, bool, NoneType,
1905
datetime, struct_time)
1906
def normalize_feedparser_dict(fp_dict):
1907
"""Convert FeedParserDict objects to normal dictionaries."""
1909
for key, value in fp_dict.items():
1910
if isinstance(value, feedparser.FeedParserDict):
1911
value = normalize_feedparser_dict(value)
1912
elif isinstance(value, dict):
1913
value = dict((_convert_if_feedparser_dict(k),
1914
_convert_if_feedparser_dict(v)) for (k, v) in
1916
elif isinstance(value, list):
1917
value = [_convert_if_feedparser_dict(o) for o in value]
1918
elif isinstance(value, tuple):
1919
value = tuple(_convert_if_feedparser_dict(o) for o in value)
1921
if not value.__class__ in _simple_feedparser_values:
1922
raise ValueError("Can't normalize: %r (%s)" %
1923
(value, value.__class__))
1927
def _convert_if_feedparser_dict(obj):
1928
if isinstance(obj, feedparser.FeedParserDict):
1929
return normalize_feedparser_dict(obj)
1934
for o in objectList:
1935
if o.classString in ('item', 'file-item'):
1936
entry = o.savedData.pop('entry')
1937
fp_values = FeedParserValues(entry)
1938
o.savedData.update(fp_values.data)
1939
o.savedData['feedparser_output'] = fp_values.normalized_entry
1943
def upgrade76(objectList):
1945
for o in objectList:
1946
if o.classString == 'feed':
1947
feed_impl = o.savedData['actualFeed']
1948
o.savedData['visible'] = feed_impl.savedData.pop('visible')
1952
def upgrade77(objectList):
1953
"""Drop ufeed and actualFeed attributes, replace them with id
1958
for o in objectList:
1959
last_id = max(o.savedData['id'], last_id)
1960
if o.classString == 'feed':
1963
next_id = last_id + 1
1965
feed_impl = feed.savedData['actualFeed']
1966
feed_impl.savedData['ufeed_id'] = feed.savedData['id']
1967
feed.savedData['feed_impl_id'] = feed_impl.savedData['id'] = next_id
1968
del feed_impl.savedData['ufeed']
1969
del feed.savedData['actualFeed']
1971
changed.add(feed_impl)
1972
objectList.append(feed_impl)
1976
def upgrade78(objectList):
1977
"""Drop iconCache attribute. Replace it with icon_cache_id. Make
1978
IconCache objects into top-level entities.
1982
icon_cache_containers = []
1984
for o in objectList:
1985
last_id = max(o.savedData['id'], last_id)
1986
if o.classString in ('feed', 'item', 'file-item', 'channel-guide'):
1987
icon_cache_containers.append(o)
1989
next_id = last_id + 1
1990
for obj in icon_cache_containers:
1991
icon_cache = obj.savedData['iconCache']
1992
if icon_cache is not None:
1993
obj.savedData['icon_cache_id'] = icon_cache.savedData['id'] = next_id
1994
changed.add(icon_cache)
1995
objectList.append(icon_cache)
1997
obj.savedData['icon_cache_id'] = None
1998
del obj.savedData['iconCache']
2003
def upgrade79(objectList):
2004
"""Convert RemoteDownloader.status from SchemaSimpleContainer to
2005
SchemaReprContainer.
2008
def convert_to_repr(obj, key):
2009
obj.savedData[key] = repr(obj.savedData[key])
2012
for o in objectList:
2013
if o.classString == 'remote-downloader':
2014
convert_to_repr(o, 'status')
2015
elif o.classString in ('item', 'file-item'):
2016
convert_to_repr(o, 'feedparser_output')
2017
elif o.classString == 'scraper-feed-impl':
2018
convert_to_repr(o, 'linkHistory')
2019
elif o.classString in ('rss-multi-feed-impl', 'search-feed-impl'):
2020
convert_to_repr(o, 'etag')
2021
convert_to_repr(o, 'modified')
2022
elif o.classString in ('playlist', 'playlist-folder'):
2023
convert_to_repr(o, 'item_ids')
2024
elif o.classString == 'taborder-order':
2025
convert_to_repr(o, 'tab_ids')
2026
elif o.classString == 'channel-guide':
2027
convert_to_repr(o, 'allowedURLs')
2028
elif o.classString == 'theme-history':
2029
convert_to_repr(o, 'pastThemes')
2030
elif o.classString == 'widgets-frontend-state':
2031
convert_to_repr(o, 'list_view_displays')
2035
# There is no upgrade80. That version was the version we switched how
2036
# the database was stored.
2038
def upgrade81(cursor):
2039
"""Add the was_downloaded column to downloader."""
2042
cursor.execute("ALTER TABLE remote_downloader ADD state TEXT")
2044
for row in cursor.execute("SELECT id, status FROM remote_downloader"):
2046
status = eval(row[1], __builtins__,
2047
{'datetime': datetime, 'time': time})
2048
state = status.get('state', u'downloading')
2049
to_update.append((id, state))
2050
for id, state in to_update:
2051
cursor.execute("UPDATE remote_downloader SET state=? WHERE id=?",
2054
def upgrade82(cursor):
2055
"""Add the state column to item."""
2056
cursor.execute("ALTER TABLE item ADD was_downloaded INTEGER")
2057
cursor.execute("ALTER TABLE file_item ADD was_downloaded INTEGER")
2058
cursor.execute("UPDATE file_item SET was_downloaded=0")
2061
for row in cursor.execute("SELECT id, downloader_id, expired FROM item"):
2062
if row[1] is not None or row[2]:
2063
# item has a downloader, or was expired, either way it was
2064
# downloaded at some point.
2065
downloaded.append(row[0])
2066
cursor.execute("UPDATE item SET was_downloaded=0")
2067
# sqlite can only handle 999 variables at once, which can be less
2068
# then the number of downloaded items (#11717). Let's go for
2069
# chunks of 500 at a time to be safe.
2070
for start_pos in xrange(0, len(downloaded), 500):
2071
downloaded_chunk = downloaded[start_pos:start_pos+500]
2072
placeholders = ', '.join('?' for i in xrange(len(downloaded_chunk)))
2073
cursor.execute("UPDATE item SET was_downloaded=1 "
2074
"WHERE id IN (%s)" % placeholders, downloaded_chunk)
2076
def upgrade83(cursor):
2077
"""Merge the items and file_items tables together."""
2079
cursor.execute("ALTER TABLE item ADD is_file_item INTEGER")
2080
cursor.execute("ALTER TABLE item ADD filename TEXT")
2081
cursor.execute("ALTER TABLE item ADD deleted INTEGER")
2082
cursor.execute("ALTER TABLE item ADD shortFilename TEXT")
2083
cursor.execute("ALTER TABLE item ADD offsetPath TEXT")
2084
# Set values for existing Item objects
2085
cursor.execute("UPDATE item SET is_file_item=0, filename=NULL, "
2086
"deleted=NULL, shortFilename=NULL, offsetPath=NULL")
2087
# Set values for FileItem objects coming from the file_items table
2088
columns = ('id', 'feed_id', 'downloader_id', 'parent_id', 'seen',
2089
'autoDownloaded', 'pendingManualDL', 'pendingReason', 'title',
2090
'expired', 'keep', 'creationTime', 'linkNumber', 'icon_cache_id',
2091
'downloadedTime', 'watchedTime', 'isContainerItem',
2092
'videoFilename', 'isVideo', 'releaseDateObj',
2093
'eligibleForAutoDownload', 'duration', 'screenshot', 'resumeTime',
2094
'channelTitle', 'license', 'rss_id', 'thumbnail_url',
2095
'entry_title', 'raw_descrption', 'link', 'payment_link',
2096
'comments_link', 'url', 'enclosure_size', 'enclosure_type',
2097
'enclosure_format', 'feedparser_output', 'was_downloaded',
2098
'filename', 'deleted', 'shortFilename', 'offsetPath',)
2099
columns_connected = ', '.join(columns)
2100
cursor.execute('INSERT INTO item (is_file_item, %s) '
2101
'SELECT 1, %s FROM file_item' % (columns_connected,
2103
cursor.execute("DROP TABLE file_item")
2105
def upgrade84(cursor):
2106
"""Fix "field_impl" typo"""
2107
cursor.execute("ALTER TABLE field_impl RENAME TO feed_impl")
2109
def upgrade85(cursor):
2110
"""Set seen attribute for container items"""
2112
cursor.execute("UPDATE item SET seen=0 WHERE isContainerItem")
2113
cursor.execute("UPDATE item SET seen=1 "
2114
"WHERE isContainerItem AND NOT EXISTS "
2115
"(SELECT 1 FROM item AS child WHERE "
2116
"child.parent_id=item.id AND NOT child.seen)")
2118
def upgrade86(cursor):
2119
"""Move the lastViewed attribute from feed_impl to feed."""
2120
cursor.execute("ALTER TABLE feed ADD last_viewed TIMESTAMP")
2122
feed_impl_tables = ('feed_impl', 'rss_feed_impl', 'rss_multi_feed_impl',
2123
'scraper_feed_impl', 'search_feed_impl',
2124
'directory_watch_feed_impl', 'directory_feed_impl',
2125
'search_downloads_feed_impl', 'manual_feed_impl',
2126
'single_feed_impl',)
2127
selects = ['SELECT ufeed_id, lastViewed FROM %s' % table \
2128
for table in feed_impl_tables]
2129
union = ' UNION '.join(selects)
2130
cursor.execute("UPDATE feed SET last_viewed = "
2131
"(SELECT lastViewed FROM (%s) WHERE ufeed_id = feed.id)" % union)
2133
def upgrade87(cursor):
2134
"""Make last_viewed a "timestamp" column rather than a "TIMESTAMP"
2136
# see 11716 for details
2138
columns_with_type = []
2139
cursor.execute("PRAGMA table_info('feed')")
2140
for column_info in cursor.fetchall():
2141
column = column_info[1]
2142
type = column_info[2]
2143
columns.append(column)
2144
if type == 'TIMESTAMP':
2147
type += ' PRIMARY KEY'
2148
columns_with_type.append("%s %s" % (column, type))
2149
cursor.execute("ALTER TABLE feed RENAME TO old_feed")
2150
cursor.execute("CREATE TABLE feed (%s)" % ', '.join(columns_with_type))
2151
cursor.execute("INSERT INTO feed (%s) SELECT %s FROM old_feed" %
2152
(', '.join(columns), ', '.join(columns)))
2153
cursor.execute("DROP TABLE old_feed")
2155
def upgrade88(cursor):
2156
"""Replace playlist.item_ids, with PlaylistItemMap objects."""
2158
id_counter = itertools.count(get_next_id(cursor))
2161
for table_name in ('playlist_item_map', 'playlist_folder_item_map'):
2162
cursor.execute("SELECT COUNT(*) FROM sqlite_master "
2163
"WHERE name=? and type='table'", (table_name,))
2164
if cursor.fetchone()[0] > 0:
2165
logging.warn("dropping %s in upgrade88", table_name)
2166
cursor.execute("DROP TABLE %s " % table_name)
2167
cursor.execute("CREATE TABLE playlist_item_map (id integer PRIMARY KEY, "
2168
"playlist_id integer, item_id integer, position integer)")
2169
cursor.execute("CREATE TABLE playlist_folder_item_map "
2170
"(id integer PRIMARY KEY, playlist_id integer, item_id integer, "
2171
" position integer, count integer)")
2173
sql = "SELECT id, folder_id, item_ids FROM playlist"
2174
for row in list(cursor.execute(sql)):
2175
id, folder_id, item_ids = row
2176
item_ids = eval(item_ids, {}, {})
2177
for i, item_id in enumerate(item_ids):
2178
cursor.execute("INSERT INTO playlist_item_map "
2179
"(id, item_id, playlist_id, position) VALUES (?, ?, ?, ?)",
2180
(id_counter.next(), item_id, id, i))
2181
if folder_id is not None:
2182
if folder_id not in folder_count:
2183
folder_count[folder_id] = {}
2185
folder_count[folder_id][item_id] += 1
2187
folder_count[folder_id][item_id] = 1
2189
sql = "SELECT id, item_ids FROM playlist_folder"
2190
for row in list(cursor.execute(sql)):
2192
item_ids = eval(item_ids, {}, {})
2193
this_folder_count = folder_count[id]
2194
for i, item_id in enumerate(item_ids):
2196
count = this_folder_count[item_id]
2198
# item_id is listed for this playlist folder, but none
2199
# of it's child folders. It's not clear how it
2200
# happened, but forget about it. (#12301)
2202
cursor.execute("INSERT INTO playlist_folder_item_map "
2203
"(id, item_id, playlist_id, position, count) "
2204
"VALUES (?, ?, ?, ?, ?)",
2205
(id_counter.next(), item_id, id, i, count))
2206
for table in ('playlist_folder', 'playlist'):
2207
remove_column(cursor, table, ['item_id'])
2209
def upgrade89(cursor):
2210
"""Set videoFilename column for downloaded items."""
2212
from miro.plat.utils import filenameToUnicode
2214
# for Items, calculate from the downloader
2215
for row in cursor.execute("SELECT id, downloader_id FROM item "
2216
"WHERE NOT is_file_item AND videoFilename = ''").fetchall():
2217
item_id, downloader_id = row
2218
if downloader_id is None:
2220
cursor.execute("SELECT state, status FROM remote_downloader "
2221
"WHERE id=?", (downloader_id,))
2222
results = cursor.fetchall()
2223
if len(results) == 0:
2224
cursor.execute("SELECT origURL FROM feed "
2225
"JOIN item ON item.feed_id=feed.id "
2226
"WHERE item.id=?", (item_id,))
2227
row = cursor.fetchall()[0]
2228
if row[0] == 'dtv:manualFeed':
2229
# external download, let's just delete the row.
2230
cursor.execute("DELETE FROM item WHERE id=?", (item_id,))
2232
cursor.execute("UPDATE item "
2233
"SET downloader_id=NULL, seen=NULL, keep=NULL, "
2234
"pendingManualDL=0, filename=NULL, watchedTime=NULL, "
2235
"duration=NULL, screenshot=NULL, "
2236
"isContainerItem=NULL, expired=1 "
2244
status = eval(status, __builtins__, {'datetime': datetime})
2245
filename = status.get('filename')
2246
if (state in ('stopped', 'finished', 'uploading', 'uploading-paused')
2248
filename = filenameToUnicode(filename)
2249
cursor.execute("UPDATE item SET videoFilename=? WHERE id=?",
2250
(filename, item_id))
2251
# for FileItems, just copy from filename
2252
cursor.execute("UPDATE item set videoFilename=filename "
2253
"WHERE is_file_item")
2255
def upgrade90(cursor):
2256
"""Add the was_downloaded column to downloader."""
2257
cursor.execute("ALTER TABLE remote_downloader ADD main_item_id integer")
2258
for row in cursor.execute("SELECT id FROM remote_downloader").fetchall():
2259
downloader_id = row[0]
2260
cursor.execute("SELECT id FROM item WHERE downloader_id=? LIMIT 1",
2262
row = cursor.fetchone()
2264
# set main_item_id to one of the item ids, it doesn't matter which
2266
cursor.execute("UPDATE remote_downloader SET main_item_id=? "
2267
"WHERE id=?", (item_id, downloader_id))
2269
# no items for a downloader, delete the downloader
2270
cursor.execute("DELETE FROM remote_downloader WHERE id=?",
2273
def upgrade91(cursor):
2274
"""Add lots of indexes."""
2275
cursor.execute("CREATE INDEX item_feed ON item (feed_id)")
2276
cursor.execute("CREATE INDEX item_downloader ON item (downloader_id)")
2277
cursor.execute("CREATE INDEX item_feed_downloader ON item "
2278
"(feed_id, downloader_id)")
2279
cursor.execute("CREATE INDEX downloader_state ON remote_downloader (state)")
2281
def upgrade92(cursor):
2282
feed_impl_tables = ('feed_impl', 'rss_feed_impl', 'rss_multi_feed_impl',
2283
'scraper_feed_impl', 'search_feed_impl',
2284
'directory_watch_feed_impl', 'directory_feed_impl',
2285
'search_downloads_feed_impl', 'manual_feed_impl',
2286
'single_feed_impl',)
2287
for table in feed_impl_tables:
2288
remove_column(cursor, table, ['lastViewed'])
2289
remove_column(cursor, 'playlist', ['item_ids'])
2290
remove_column(cursor, 'playlist_folder', ['item_ids'])
2292
def upgrade93(cursor):
2293
VIDEO_EXTENSIONS = ['.mov', '.wmv', '.mp4', '.m4v', '.ogv', '.anx',
2294
'.mpg', '.avi', '.flv', '.mpeg', '.divx', '.xvid',
2295
'.rmvb', '.mkv', '.m2v', '.ogm']
2296
AUDIO_EXTENSIONS = ['.mp3', '.m4a', '.wma', '.mka', '.ogg', '.flac']
2298
video_filename_expr = '(%s)' % ' OR '.join("videoFilename LIKE '%%%s'" % ext
2299
for ext in VIDEO_EXTENSIONS)
2301
audio_filename_expr = '(%s)' % ' OR '.join("videoFilename LIKE '%%%s'" % ext
2302
for ext in AUDIO_EXTENSIONS)
2304
cursor.execute("ALTER TABLE item ADD file_type text")
2305
cursor.execute("CREATE INDEX item_file_type ON item (file_type)")
2306
cursor.execute("UPDATE item SET file_type = 'video' "
2307
"WHERE " + video_filename_expr)
2308
cursor.execute("UPDATE item SET file_type = 'audio' "
2309
"WHERE " + audio_filename_expr)
2310
cursor.execute("UPDATE item SET file_type = 'other' "
2311
"WHERE file_type IS NULL AND videoFilename IS NOT NULL AND "
2312
"videoFilename != ''")
2314
def upgrade94(cursor):
2315
cursor.execute("UPDATE item SET downloadedTime=NULL "
2316
"WHERE deleted OR downloader_id IS NULL")
2318
def upgrade95(cursor):
2319
"""Delete FileItem objects that are duplicates of torrent files. (#11818)
2321
cursor.execute("SELECT item.id, item.videoFilename, rd.status "
2323
"JOIN remote_downloader rd ON item.downloader_id=rd.id "
2324
"WHERE rd.state in ('stopped', 'finished', 'uploading', "
2325
"'uploading-paused')")
2326
for row in cursor.fetchall():
2327
id, videoFilename, status = row
2328
status = eval(status, __builtins__,
2329
{'datetime': datetime, 'time': time})
2330
if (videoFilename and videoFilename != status.get('filename')):
2331
pathname = os.path.join(status.get('filename'), videoFilename)
2332
# Here's the situation: We downloaded a torrent and that
2333
# torrent had a single video as it's child. We then made
2334
# the torrent's videoFilename be the path to that video
2335
# instead of creating a new FileItem. This is broken for
2336
# a bunch of reasons, so we're getting rid of it. Undo
2337
# the trickyness that we did and delete any duplicate
2338
# items that may have been created. The next update will
2339
# remove the videoFilename column.
2340
cursor.execute("DELETE FROM item "
2341
"WHERE is_file_item AND videoFilename =?", (pathname,))
2342
cursor.execute("UPDATE item "
2343
"SET file_type='other', isContainerItem=1 "
2344
"WHERE id=?", (id,))
2346
def upgrade96(cursor):
2347
"""Delete the videoFilename and isVideo column."""
2348
remove_column(cursor, 'item', ['videoFilename', 'isVideo'])
2350
def upgrade97(cursor):
2351
"""Add another indexes, this is make tab switching faster.
2353
cursor.execute("CREATE INDEX item_feed_visible ON item (feed_id, deleted)")
2355
def upgrade98(cursor):
2356
"""Add an index for item parents
2358
cursor.execute("CREATE INDEX item_parent ON item (parent_id)")
2360
def upgrade99(cursor):
2361
"""Set the filename attribute for downloaded Item objects
2363
from miro.plat.utils import filenameToUnicode
2364
cursor.execute("SELECT id, status from remote_downloader "
2365
"WHERE state in ('stopped', 'finished', 'uploading', "
2366
"'uploading-paused')")
2367
for row in cursor.fetchall():
2368
downloader_id = row[0]
2369
status = eval(row[1], __builtins__,
2370
{'datetime': datetime, 'time': time})
2371
filename = status.get('filename')
2373
filename = filenameToUnicode(filename)
2374
cursor.execute("UPDATE item SET filename=? WHERE downloader_id=?",
2375
(filename, downloader_id))
2377
class TimeModuleShadow:
2378
"""In Python 2.6, time.struct_time is a named tuple and evals
2379
poorly, so we have struct_time_shadow which takes the arguments
2380
that struct_time should have and returns a 9-tuple
2382
def struct_time(self, tm_year=0, tm_mon=0, tm_mday=0, tm_hour=0,
2383
tm_min=0, tm_sec=0, tm_wday=0, tm_yday=0, tm_isdst=0):
2384
return (tm_year, tm_mon, tm_mday, tm_hour, tm_min, tm_sec,
2385
tm_wday, tm_yday, tm_isdst)
2387
_TIME_MODULE_SHADOW = TimeModuleShadow()
2389
def eval_container(repr):
2390
"""Convert a column that's stored using repr to a python
2392
return eval(repr, __builtins__, {'datetime': datetime,
2393
'time': _TIME_MODULE_SHADOW})
2395
def upgrade100(cursor):
2396
"""Adds the Miro audio guide as a site for anyone who doesn't
2397
already have it and isn't using a theme.
2399
# if the user is using a theme, we don't do anything
2400
if not config.get(prefs.THEME_NAME) == prefs.THEME_NAME.default:
2403
audio_guide_url = u'https://www.miroguide.com/audio/'
2404
favicon_url = u'https://www.miroguide.com/favicon.ico'
2405
cursor.execute("SELECT count(*) FROM channel_guide WHERE url=?",
2407
count = cursor.fetchone()[0]
2411
next_id = get_next_id(cursor)
2413
cursor.execute("INSERT INTO channel_guide "
2414
"(id, url, allowedURLs, updated_url, favicon, firstTime) VALUES (?, ?, ?, ?, ?, ?)",
2415
(next_id, audio_guide_url, "[]", audio_guide_url,
2418
# add the new Audio Guide to the site tablist
2419
cursor.execute('SELECT tab_ids FROM taborder_order WHERE type=?',
2421
row = cursor.fetchone()
2424
tab_ids = eval_container(row[0])
2425
except StandardError:
2427
tab_ids.append(next_id)
2428
cursor.execute('UPDATE taborder_order SET tab_ids=? WHERE type=?',
2429
(repr(tab_ids), 'site'))
2431
# no site taborder (#11985). We will create the TabOrder
2432
# object on startup, so no need to do anything here
2435
def upgrade101(cursor):
2436
"""For torrent folders where a child item has been deleted, change
2437
the state from 'stopped' to 'finished' and set child_deleted to
2440
cursor.execute("ALTER TABLE remote_downloader ADD child_deleted INTEGER")
2441
cursor.execute("UPDATE remote_downloader SET child_deleted = 0")
2442
cursor.execute("SELECT id, status FROM remote_downloader "
2443
"WHERE state = 'stopped'")
2444
for row in cursor.fetchall():
2447
status = eval_container(status)
2448
except StandardError:
2449
# Not sure what to do here. I think ignoring is not
2450
# ideal, but won't result in anything too bad (BDK)
2452
if status['endTime'] == status['startTime']:
2453
# For unfinished downloads, unset the filename which got
2455
cursor.execute("UPDATE item SET filename=NULL "
2456
"WHERE downloader_id=?", (id,))
2457
elif status['dlerType'] != 'BitTorrent':
2458
status['state'] = 'finished'
2459
cursor.execute("UPDATE remote_downloader "
2460
"SET state='finished', child_deleted=1, status=? "
2461
"WHERE id=?", (repr(status), id))
2463
def upgrade102(cursor):
2464
"""Fix for the embarrasing bug in upgrade101
2466
This statement was exactly the opposite of what we want::
2468
elif status['dlerType'] != 'BitTorrent':
2470
cursor.execute("SELECT id, status, child_deleted FROM remote_downloader "
2471
"WHERE state = 'stopped'")
2472
for row in cursor.fetchall():
2473
id, status, child_deleted = row
2474
status = eval_container(status)
2475
if status['dlerType'] != 'BitTorrent' and child_deleted:
2476
# I don't think that it's actually possible, but fix HTTP
2477
# downloaders that were changed in upgrade101
2478
status['state'] = 'stopped'
2479
cursor.execute("UPDATE remote_downloader "
2480
"SET state='stopped', child_deleted=0, status=? "
2481
"WHERE id=?", (repr(status), id))
2482
elif (status['endTime'] != status['startTime'] and
2483
status['dlerType'] == 'BitTorrent'):
2484
# correctly execute what upgrade101 was trying to do
2485
status['state'] = 'finished'
2486
cursor.execute("UPDATE remote_downloader "
2487
"SET state='finished', child_deleted=1, status=? "
2488
"WHERE id=?", (repr(status), id))
2490
def upgrade103(cursor):
2491
"""Possible fix for #11730.
2493
Delete downloaders with duplicate origURL values.
2495
cursor.execute("SELECT MIN(id), origURL FROM remote_downloader "
2497
"HAVING count(*) > 1")
2498
for row in cursor.fetchall():
2500
cursor.execute("SELECT id FROM remote_downloader "
2501
"WHERE origURL=? and id != ?", (origURL, id_))
2502
for row in cursor.fetchall():
2504
cursor.execute("UPDATE item SET downloader_id=? "
2505
"WHERE downloader_id=?", (id_, dup_id))
2506
cursor.execute("DELETE FROM remote_downloader WHERE id=?",
2509
def upgrade104(cursor):
2510
cursor.execute("UPDATE item SET seen=0 WHERE seen IS NULL")
2511
cursor.execute("UPDATE item SET keep=0 WHERE keep IS NULL")
2513
def upgrade105(cursor):
2514
"""Move metainfo and fastResumeData out of the status dict."""
2516
cursor.execute("ALTER TABLE remote_downloader ADD metainfo BLOB")
2517
cursor.execute("ALTER TABLE remote_downloader ADD fast_resume_data BLOB")
2519
cursor.execute("SELECT id, status FROM remote_downloader")
2520
for row in cursor.fetchall():
2521
id, status_repr = row
2523
status = eval_container(status_repr)
2524
except StandardError:
2526
metainfo = status.pop('metainfo', None)
2527
fast_resume_data = status.pop('fastResumeData', None)
2528
new_status = repr(status)
2529
if metainfo is not None:
2530
metainfo_value = buffer(metainfo)
2532
metainfo_value = None
2533
if fast_resume_data is not None:
2534
fast_resume_data_value = buffer(fast_resume_data)
2536
fast_resume_data_value = None
2537
cursor.execute("UPDATE remote_downloader "
2538
"SET status=?, metainfo=?, fast_resume_data=? "
2540
(new_status, metainfo_value, fast_resume_data_value, id))
2543
def upgrade106(cursor):
2544
tables = get_object_tables(cursor)
2545
# figure out which ids, if any are duplicated
2546
id_union = ' UNION ALL '.join(['SELECT id FROM %s' % t for t in tables])
2547
cursor.execute("SELECT count(*) as id_count, id FROM (%s) "
2548
"GROUP BY id HAVING id_count > 1" % id_union)
2549
duplicate_ids = set([r[1] for r in cursor])
2550
if len(duplicate_ids) == 0:
2553
id_counter = itertools.count(get_next_id(cursor))
2555
def update_value(table, column, old_value, new_value):
2556
cursor.execute("UPDATE %s SET %s=%s WHERE %s=%s" % (table, column,
2557
new_value, column, old_value))
2559
for table in tables:
2561
# let feed objects keep their id, it's fairly annoying to
2562
# have to update the ufeed atribute for all the FeedImpl
2563
# subclasses. The id won't be a duplicate anymore once we
2564
# update the other tables
2566
cursor.execute("SELECT id FROM %s" % table)
2567
for row in cursor.fetchall():
2569
if id in duplicate_ids:
2570
new_id = id_counter.next()
2571
# assign a new id to the object
2572
update_value(table, 'id', id, new_id)
2574
if table == 'icon_cache':
2575
update_value('item', 'icon_cache_id', id, new_id)
2576
update_value('feed', 'icon_cache_id', id, new_id)
2577
update_value('channel_guide', 'icon_cache_id', id, new_id)
2578
elif table.endswith('feed_impl'):
2579
update_value('feed', 'feed_impl_id', id, new_id)
2580
elif table == 'channel_folder':
2581
update_value('feed', 'folder_id', id, new_id)
2582
elif table == 'remote_downloader':
2583
update_value('item', 'downloader_id', id, new_id)
2584
elif table == 'item_id':
2585
update_value('item', 'parent_id', id, new_id)
2586
update_value('downloader', 'main_item_id', id, new_id)
2587
update_value('playlist_item_map', 'item_id', id, new_id)
2588
update_value('playlist_folder_item_map', 'item_id', id,
2590
elif table == 'playlist_folder':
2591
update_value('playlist', 'folder_id', id, new_id)
2592
elif table == 'playlist':
2593
update_value('playlist_item_map', 'playlist_id', id, new_id)
2594
update_value('playlist_folder_item_map', 'playlist_id',
2596
# note we don't handle TabOrder.tab_ids here. That's
2597
# because it's a list of ids, so it's hard to fix
2598
# using SQL. Also, the TabOrder code is able to
2599
# recover from missing/extra ids in its list. The
2600
# only bad thing that will happen is the user's tab
2601
# order will be changed.
2603
def upgrade107(cursor):
2604
cursor.execute("CREATE TABLE db_log_entry ("
2605
"id integer, timestamp real, priority integer, description text)")
2607
def upgrade108(cursor):
2608
"""Drop the feedparser_output column from item.
2610
remove_column(cursor, "item", ["feedparser_output"])
2612
def upgrade109(cursor):
2613
"""Add the media_type_checked column to item """
2614
cursor.execute("ALTER TABLE item ADD media_type_checked integer")
2615
cursor.execute("UPDATE item SET media_type_checked=0")
2617
def upgrade110(cursor):
2618
"""Make set last_viewed on the manual feed to datetime.max"""
2619
cursor.execute("select ufeed_id from manual_feed_impl")
2621
cursor.execute("UPDATE feed SET last_viewed=? WHERE id=?",
2622
(datetime.datetime.max, row[0]))
added
removed
portable/databaseupgrade.py
