← Back to branch summary

~widelands-dev/widelands-website/django_staticfiles

« back to all changes in this revision

Viewing changes to pybb/management/commands/import_punbb.py

  • Committer: Holger Rapp
  • Date: 2009-02-25 16:55:36 UTC
  • Revision ID: sirver@kallisto.local-20090225165536-3abfhjx8qsgtzyru
- Added my hacked version of pybb. Remerging new versions is very difficult at this point :(

Show diffs side-by-side

added added

removed removed

Lines of Context:
pybb/management/commands/import_punbb.py
 
1
import sqlalchemy as SA
 
2
from sqlalchemy import sql
 
3
from datetime import datetime
 
4
import os
 
5
from optparse import make_option
 
6
 
 
7
from django.core.management.base import BaseCommand, CommandError
 
8
from django.contrib.auth.models import User
 
9
 
 
10
from pybb.models import Category, Forum, Topic, Post, Profile
 
11
from pybb.lib import phpserialize
 
12
 
 
13
class Command(BaseCommand):
 
14
 
 
15
    option_list = BaseCommand.option_list + (
 
16
        make_option('--user', help=u'Punbb DB username'),
 
17
        make_option('--password', help=u'Punbb DB password'),
 
18
        make_option('--host', default='localhost', help=u'Punbb DB host'),
 
19
        make_option('--port', help=u'Punbb DB port'),
 
20
        make_option('--encoding', default='cp1251', help=u'Punbb DB encoding'),
 
21
        make_option('--mysql-encoding', help=u'Punbb DB encoding. I can\'t explain this yet'),
 
22
        make_option('--engine', default='mysql', help=u'Punbb DB engine [postgres, mysql etc]'),
 
23
        make_option('--prefix', default='punbb_', help=u'Punbb DB tables prefix'),
 
24
    )
 
25
    help = u'Imports Punbb database. Attention: old contents of pybb database will be removed'
 
26
    args = '<db name>'
 
27
 
 
28
    def handle(self, *args, **options):
 
29
        if len(args) != 1:
 
30
            raise CommandError('Punbb database name required')
 
31
        else:
 
32
            DBNAME = args[0]
 
33
        ENCODING = options['encoding']
 
34
        MYSQL_ENCODING = options['mysql_encoding']
 
35
        PREFIX = options['prefix']
 
36
 
 
37
        uri = '%s://' % options['engine']
 
38
        if options['user'] is not None:
 
39
            uri += options['user']
 
40
        if options['password'] is not None:
 
41
            uri += ':%s' % options['password']
 
42
        if options['host'] is not None:
 
43
            uri += '@%s' % options['host']
 
44
        if options['port'] is not None:
 
45
            uri += ':%s' % options['port']
 
46
        uri += '/%s' % DBNAME
 
47
 
 
48
        if options['engine'] == 'mysql' and not MYSQL_ENCODING:
 
49
            uri += '?charset=%s' % ENCODING.replace('-', '')
 
50
 
 
51
        engine = SA.create_engine(uri, convert_unicode=False)
 
52
        conn = engine.connect()
 
53
 
 
54
        meta = SA.MetaData()
 
55
        meta.bind = engine
 
56
 
 
57
        users_table = SA.Table(PREFIX + 'users', meta, autoload=True)
 
58
        cats_table = SA.Table(PREFIX + 'categories', meta, autoload=True)
 
59
        forums_table = SA.Table(PREFIX + 'forums', meta, autoload=True)
 
60
        topics_table = SA.Table(PREFIX + 'topics', meta, autoload=True)
 
61
        posts_table = SA.Table(PREFIX + 'posts', meta, autoload=True)
 
62
        groups_table = SA.Table(PREFIX + 'groups', meta, autoload=True)
 
63
        config_table = SA.Table(PREFIX + 'config', meta, autoload=True)
 
64
        subscriptions_table = SA.Table(PREFIX + 'subscriptions', meta, autoload=True)
 
65
 
 
66
        def decode(data):
 
67
            if data is None:
 
68
                return None
 
69
            if options['engine'] != 'mysql' or MYSQL_ENCODING:
 
70
                return data.decode(ENCODING, 'replace')
 
71
            else:
 
72
                return data
 
73
 
 
74
        # Import begins
 
75
        print 'Searching admin group'
 
76
 
 
77
        ADMIN_GROUP = None
 
78
        for count, row in enumerate(conn.execute(sql.select([groups_table]))):
 
79
            if row['g_title'] == 'Administrators':
 
80
                print 'Admin group was found'
 
81
                ADMIN_GROUP = row['g_id']
 
82
 
 
83
        if ADMIN_GROUP is None:
 
84
            print 'Admin group was NOT FOUND'
 
85
 
 
86
 
 
87
        print 'Importing users'
 
88
        users = {}
 
89
        User.objects.all().delete()
 
90
 
 
91
        count = 0
 
92
        for count, row in enumerate(conn.execute(sql.select([users_table]))):
 
93
            joined = datetime.fromtimestamp(row['registered'])
 
94
            last_login = datetime.fromtimestamp(row['last_visit'])
 
95
            if len(row['password']) == 40:
 
96
                hash = 'sha1$$' + row['password']
 
97
            else:
 
98
                hash = 'md5$$' + row['password']
 
99
            user = User(username=decode(row['username']),
 
100
                        email=row['email'],
 
101
                        first_name=decode((row['realname'] or '')[:30]),
 
102
                        date_joined=joined,
 
103
                        last_login=last_login,
 
104
                        password=hash
 
105
                        )
 
106
            if row['group_id'] == ADMIN_GROUP:
 
107
                print u'Admin was found: %s' % row['username']
 
108
                user.is_superuser = True
 
109
                user.is_staff = True
 
110
 
 
111
            try:
 
112
                user.save()
 
113
            except Exception, ex:
 
114
                print ex
 
115
            else:
 
116
                users[row['id']] = user
 
117
 
 
118
                profile = user.pybb_profile
 
119
                profile.jabber = decode(row['jabber'])
 
120
                profile.icq = decode(row['icq'])
 
121
                profile.yahoo = decode(row['yahoo'])
 
122
                profile.msn = decode(row['msn'])
 
123
                profile.aim = decode(row['aim'])
 
124
                profile.location = decode(row['location'])
 
125
                profile.signature = decode(row['signature'])
 
126
                profile.show_signatures = bool(row['show_sig'])
 
127
                profile.time_zone = row['timezone']
 
128
 
 
129
        print 'Total: %d' % (count + 1)
 
130
        print 'Imported: %d' % len(users)
 
131
        print
 
132
 
 
133
        print 'Importing categories'
 
134
        cats = {}
 
135
        Category.objects.all().delete()
 
136
 
 
137
        count = 0
 
138
        for count, row in enumerate(conn.execute(sql.select([cats_table]))):
 
139
            cat = Category(name=decode(row['cat_name']),
 
140
                           position=row['disp_position'])
 
141
            cat.save()
 
142
            cats[row['id']] = cat
 
143
 
 
144
        print 'Total: %d' % (count + 1)
 
145
        print 'Imported: %d' % len(cats)
 
146
        print
 
147
 
 
148
        print 'Importing forums'
 
149
        forums = {}
 
150
        moderators = {}
 
151
        Forum.objects.all().delete()
 
152
 
 
153
        count = 0
 
154
        for count, row in enumerate(conn.execute(sql.select([forums_table]))):
 
155
            if row['last_post']:
 
156
                updated = datetime.fromtimestamp(row['last_post'])
 
157
            else:
 
158
                updated = None
 
159
 
 
160
            forum = Forum(name=decode(row['forum_name']),
 
161
                          position=row['disp_position'],
 
162
                          description=decode(row['forum_desc'] or ''),
 
163
                          category=cats[row['cat_id']])
 
164
            forum.save()
 
165
            forums[row['id']] = forum
 
166
 
 
167
            forum._pybb_updated = updated
 
168
 
 
169
            if row['moderators']:
 
170
                for username in phpserialize.loads(row['moderators']).iterkeys():
 
171
                    user = User.objects.get(username=username)
 
172
                    forum.moderators.add(user)
 
173
                    moderators[user.id] = user
 
174
 
 
175
        print 'Total: %d' % (count + 1)
 
176
        print 'Imported: %d' % len(forums)
 
177
        print 'Total number of moderators: %d' % len(moderators)
 
178
        print
 
179
 
 
180
 
 
181
        print 'Importing topics'
 
182
        topics = {}
 
183
        moved_count = 0
 
184
        Topic.objects.all().delete()
 
185
 
 
186
        count = 0
 
187
        for count, row in enumerate(conn.execute(sql.select([topics_table]))):
 
188
            created = datetime.fromtimestamp(row['posted'])
 
189
            updated = datetime.fromtimestamp(row['last_post'])
 
190
 
 
191
            # Skip moved topics
 
192
            if row['moved_to']:
 
193
                moved_count += 1
 
194
                continue
 
195
 
 
196
            username = decode(row['poster'])
 
197
            #testuser = users.values()[0]
 
198
            for id, testuser in users.iteritems():
 
199
                if testuser.username == username:
 
200
                    user = testuser
 
201
                    break
 
202
 
 
203
            topic = Topic(name=decode(row['subject']),
 
204
                          forum=forums[row['forum_id']],
 
205
                          views=row['num_views'],
 
206
                          sticky=bool(row['sticky']),
 
207
                          closed=bool(row['closed']),
 
208
                          user=user)
 
209
            topic.save()
 
210
            topic._pybb_updated = updated
 
211
            topic._pybb_created = created
 
212
            topic._pybb_punbb_id = row['id']
 
213
            topic._pybb_posts = 0
 
214
            #print topic._pybb_updated
 
215
            topics[row['id']] = topic
 
216
 
 
217
        print 'Total: %d' % (count + 1)
 
218
        print 'Imported: %d' % len(topics)
 
219
        print 'Moved: %d' % moved_count
 
220
        print
 
221
 
 
222
 
 
223
        print 'Importing posts'
 
224
        posts = {}
 
225
        Post.objects.all().delete()
 
226
 
 
227
        imported = 0
 
228
        count = 0
 
229
        for count, row in enumerate(conn.execute(sql.select([posts_table]))):
 
230
            created = datetime.fromtimestamp(row['posted'])
 
231
            updated = row['edited'] and datetime.fromtimestamp(row['edited']) or None
 
232
 
 
233
            if not row['poster_id'] in users:
 
234
                print 'post #%d, poster_id #%d does not exist' % (row['id'], row['poster_id'])
 
235
                continue
 
236
 
 
237
            post = Post(topic=topics[row['topic_id']],
 
238
                        created=created,
 
239
                        updated=updated,
 
240
                        markup='bbcode',
 
241
                        user=users[row['poster_id']],
 
242
                        user_ip=row['poster_ip'],
 
243
                        body=decode(row['message']))
 
244
            
 
245
            post.save()
 
246
            imported += 1
 
247
            topics[row['topic_id']]._pybb_posts += 1
 
248
 
 
249
            # Not actual hack
 
250
            ## postmarkups feels bad on some posts :-/
 
251
            #try:
 
252
                #post.save()
 
253
            #except Exception, ex:
 
254
                #print post.id, ex
 
255
                #print decode(row['message'])
 
256
                #print
 
257
            #else:
 
258
                #imported += 1
 
259
                #topics[row['topic_id']]._pybb_posts += 1
 
260
                ##posts[row['id']] = topic
 
261
 
 
262
        print 'Total: %d' % (count + 1)
 
263
        print 'Imported: %d' % imported
 
264
        print
 
265
 
 
266
 
 
267
        print 'Importing subscriptions'
 
268
 
 
269
        count = 0
 
270
        for count, row in enumerate(conn.execute(sql.select([subscriptions_table]))):
 
271
            user = users[row['user_id']]
 
272
            topic = topics[row['topic_id']]
 
273
            topic.subscribers.add(user)
 
274
 
 
275
        print 'Imported: %d' % count
 
276
 
 
277
 
 
278
        print 'Restoring topics updated and created values'
 
279
        for topic in topics.itervalues():
 
280
            topic.updated = topic._pybb_updated
 
281
            topic.created = topic._pybb_created
 
282
            topic.save()
 
283
        print
 
284
 
 
285
 
 
286
        print 'Restoring forums updated and created values'
 
287
        for forum in forums.itervalues():
 
288
            forum.updated = forum._pybb_updated
 
289
            forum.save()
 
290
        print
 
291
 
 
292
 
 
293
        print 'Importing config'
 
294
        for row in conn.execute(sql.select([config_table])):
 
295
            if row['conf_name'] == 'o_announcement_message':
 
296
                value = decode(row['conf_value'])
 
297
                if value:
 
298
                    open('pybb_announcement.txt', 'w').write(value.encode('utf-8'))
 
299
                    print 'Not empty announcement found and saved to pybb_announcement.txt'
 
300
                    print 'If you need announcement write PYBB_NOTICE = " ... text ... " to settings file'