~ubuntu-branches/ubuntu/natty/moin/natty-updates

MoinMoin - migration from 1.6.0alpha (rev 1844: 58ebb64243cc - used a similar markup as 1.5.8, but with quotes for linking stuff with blanks) to 1.6.0 (creole link style)

What it does:

a) reverse underscore == blank stuff in pagenames (introducing this was a fault)

pagename quoted pagename

-----------------------------------------------------

old MainPage/Sub_Page MainPage(2f)Sub_Page

new MainPage/Sub Page MainPage(2f)Sub(20)Page or

new MainPage/Sub_Page MainPage(2f)Sub_Page (user has to decide by editing rename1.txt)

markup

----------------------------------------------------

old MoinMoin:MainPage/Sub_Page ../Sub_Page2

new [[MoinMoin:MainPage/Sub Page]] [[../Sub Page2]]

b) decode url encoded chars in attachment names (and quote the whole fname):

markup

----------------------------------------------------

old attachment:file%20with%20blanks.txt

new [[attachment:file with blanks.txt]]

c) users: move bookmarks from separate files into user profile

d) users: generate new name[] for lists and name{} for dicts

e) kill all */MoinEditorBackup pages (replaced by drafts functionality)

@license: GNU GPL, see COPYING for details.

"""

import os.path

import re

import time

import codecs, urllib, glob

from MoinMoin import config, wikiutil

from MoinMoin.script.migration.migutil import opj, listdir, copy_file, move_file, copy_dir

import mimetypes # this MUST be after wikiutil import!

from _conv160a_wiki import convert_wiki

create_rev = True # create a <new> rev with the converted content of <new-1> rev?

def markup_converter(request, pagename, text, renames):

""" Convert the <text> content of page <pagename>, using <renames> dict

to rename links correctly. Additionally, convert some changed markup.

"""

if text.startswith('<?xml'):

# would be done with xslt processor

return text

pis, body = wikiutil.get_processing_instructions(text)

for pi, val in pis:

if pi == 'format' and val != 'wiki':

# not wiki page

return text

text = convert_wiki(request, pagename, text, renames)

return text

class EventLog:

def __init__(self, request, fname):

self.request = request

self.fname = fname

self.data = None

self.renames = {}

def read(self):

""" read complete event-log from disk """

data = []

try:

lineno = 0

f = file(self.fname, 'r')

for line in f:

lineno += 1

line = line.replace('\r', '').replace('\n', '')

if not line.strip(): # skip empty lines

continue

fields = line.split('\t')

try:

timestamp, action, kvpairs = fields[:3]

timestamp = int(timestamp)

kvdict = wikiutil.parseQueryString(kvpairs)

data.append((timestamp, action, kvdict))

except ValueError, err:

# corrupt event log line, log error and skip it

print "Error: invalid event log (%s) line %d, err: %s, SKIPPING THIS LINE!" % (self.fname, lineno, str(err))

f.close()

except IOError, err:

# no event-log

100

pass

101

self.data = data

102

103

def write(self, fname):

104

""" write complete event-log to disk """

105

if self.data:

106

f = file(fname, 'w')

107

for timestamp, action, kvdict in self.data:

108

pagename = kvdict.get('pagename')

109

if pagename and ('PAGE', pagename) in self.renames:

110

kvdict['pagename'] = self.renames[('PAGE', pagename)]

111

kvpairs = wikiutil.makeQueryString(kvdict)

112

fields = str(timestamp), action, kvpairs

113

line = '\t'.join(fields) + '\n'

114

f.write(line)

115

f.close()

116

117

def copy(self, destfname, renames):

118

self.renames = renames

119

self.read()

120

self.write(destfname)

121

122

123

class EditLog:

124

def __init__(self, request, fname):

125

self.request = request

126

self.fname = fname

127

self.data = None

128

self.renames = {}

129

130

def read(self):

131

""" read complete edit-log from disk """

132

data = {}

133

try:

134

lineno = 0

135

f = file(self.fname, 'r')

136

for line in f:

137

lineno += 1

138

line = line.replace('\r', '').replace('\n', '')

139

if not line.strip(): # skip empty lines

140

continue

141

fields = line.split('\t') + [''] * 9

142

timestamp, rev, action, pagename, ip, hostname, userid, extra, comment = fields[:9]

143

try:

144

timestamp = int(timestamp)

145

rev = int(rev)

146

except ValueError, err:

147

print "Error: %r has a damaged timestamp or revision number in log line %d [%s] - skipping this entry" % (

148

self.fname, lineno, str(err))

149

continue # ignore this line, do not terminate - to find all those errors in one go

150

pagename = wikiutil.unquoteWikiname(pagename)

151

data[(timestamp, rev, pagename)] = (timestamp, rev, action, pagename, ip, hostname, userid, extra, comment)

152

f.close()

153

except IOError, err:

154

# no edit-log

155

pass

156

self.data = data

157

158

def write(self, fname, deleted=False):

159

""" write complete edit-log to disk """

160

if self.data:

161

editlog = self.data.items()

162

editlog.sort()

163

f = file(fname, "w")

164

max_rev = 0

165

for key, fields in editlog:

166

timestamp, rev, action, pagename, ip, hostname, userid, extra, comment = fields

167

if action.startswith('ATT'):

168

try:

169

fname = urllib.unquote(extra).decode('utf-8')

170

except UnicodeDecodeError:

171

fname = urllib.unquote(extra).decode('iso-8859-1')

172

if ('FILE', pagename, fname) in self.renames:

173

fname = self.renames[('FILE', pagename, fname)]

174

extra = urllib.quote(fname.encode('utf-8'))

175

if ('PAGE', pagename) in self.renames:

176

pagename = self.renames[('PAGE', pagename)]

177

timestamp = str(timestamp)

178

if rev != 99999999:

179

max_rev = max(rev, max_rev)

180

revstr = '%08d' % rev

181

pagename = wikiutil.quoteWikinameFS(pagename)

182

fields = timestamp, revstr, action, pagename, ip, hostname, userid, extra, comment

183

log_str = '\t'.join(fields) + '\n'

184

f.write(log_str)

185

if create_rev and not deleted:

186

timestamp = str(wikiutil.timestamp2version(time.time()))

187

revstr = '%08d' % (max_rev + 1)

188

action = 'SAVE'

189

ip = '127.0.0.1'

190

hostname = 'localhost'

191

userid = ''

192

extra = ''

193

comment = "converted to 1.6 markup"

194

fields = timestamp, revstr, action, pagename, ip, hostname, userid, extra, comment

195

log_str = '\t'.join(fields) + '\n'

196

f.write(log_str)

197

f.close()

198

199

def copy(self, destfname, renames, deleted=False):

200

self.renames = renames

201

self.read()

202

self.write(destfname, deleted)

203

204

205

class PageRev:

206

""" a single revision of a page """

207

def __init__(self, request, pagename, rev_dir, rev):

208

self.request = request

209

self.pagename = pagename

210

self.rev_dir = rev_dir

211

self.rev = rev

212

213

def read(self):

214

fname = opj(self.rev_dir, '%08d' % self.rev)

215

f = file(fname, "rb")

216

data = f.read()

217

f.close()

218

data = data.decode(config.charset)

219

return data

220

221

def write(self, data, rev_dir, convert, rev=None):

222

if rev is None:

223

rev = self.rev

224

if convert:

225

data = markup_converter(self.request, self.pagename, data, self.renames)

226

fname = opj(rev_dir, '%08d' % rev)

227

data = data.encode(config.charset)

228

f = file(fname, "wb")

229

f.write(data)

230

f.close()

231

232

def copy(self, rev_dir, renames, convert=False, new_rev=None):

233

self.renames = renames

234

data = self.read()

235

self.write(data, rev_dir, convert, new_rev)

236

237

238

class Attachment:

239

""" a single attachment """

240

def __init__(self, request, attach_dir, attfile):

241

self.request = request

242

self.path = opj(attach_dir, attfile)

243

self.name = attfile.decode('utf-8', 'replace')

244

245

def copy(self, attach_dir):

246

""" copy attachment file from orig path to new destination """

247

attfile = self.name.encode('utf-8')

248

dest = opj(attach_dir, attfile)

249

copy_file(self.path, dest)

250

251

252

class Page:

253

""" represents a page with all related data """

254

def __init__(self, request, pages_dir, qpagename):

255

self.request = request

256

self.name = wikiutil.unquoteWikiname(qpagename)

257

self.name_old = self.name # renaming: still original name when self.name has the new name

258

self.page_dir = opj(pages_dir, qpagename)

259

self.current = None # int current

260

self.editlog = None # dict (see read_editlog)

261

self.revlist = None # list of ints (page text revisions)

262

self.revisions = None # dict int: pagerev obj

263

self.attachments = None # dict of unicode fname: full path

264

self.renames = {} # info for renaming pages/attachments

265

266

def read(self):

267

""" read a page, including revisions, log, attachments from disk """

268

page_dir = self.page_dir

269

# read current file

270

current_fname = opj(page_dir, 'current')

271

if os.path.exists(current_fname):

272

current_file = file(current_fname, "r")

273

current_rev = current_file.read()

274

current_file.close()

275

try:

276

self.current = int(current_rev)

277

except ValueError:

278

print "Error: invalid current file %s, SKIPPING THIS PAGE!" % current_fname

279

return

280

# read edit-log

281

editlog_fname = opj(page_dir, 'edit-log')

282

if os.path.exists(editlog_fname):

283

self.editlog = EditLog(self.request, editlog_fname)

284

# read page revisions

285

rev_dir = opj(page_dir, 'revisions')

286

if os.path.exists(rev_dir):

287

revlist = listdir(rev_dir)

288

revlist = [int(rev) for rev in revlist]

289

revlist.sort()

290

self.revlist = revlist

291

self.revisions = {}

292

for rev in revlist:

293

self.revisions[rev] = PageRev(self.request, self.name_old, rev_dir, rev)

294

# set deleted status

295

self.is_deleted = not self.revisions or self.current not in self.revisions

296

# read attachment filenames

297

attach_dir = opj(page_dir, 'attachments')

298

if os.path.exists(attach_dir):

299

self.attachments = {}

300

attlist = listdir(attach_dir)

301

for attfile in attlist:

302

a = Attachment(self.request, attach_dir, attfile)

303

self.attachments[a.name] = a

304

305

def write(self, pages_dir):

306

""" write a page, including revisions, log, attachments to disk """

307

if ('PAGE', self.name) in self.renames:

308

name_new = self.renames[('PAGE', self.name)]

309

if name_new != self.name:

310

print "Renaming page %r -> %r" % (self.name, name_new)

311

self.name_old = self.name

312

self.name = name_new

313

qpagename = wikiutil.quoteWikinameFS(self.name)

314

page_dir = opj(pages_dir, qpagename)

315

os.makedirs(page_dir)

316

# write current file

317

current = self.current

318

if current is not None:

319

if create_rev and not self.is_deleted:

320

current += 1

321

current_fname = opj(page_dir, 'current')

322

current_file = file(current_fname, "w")

323

current_str = '%08d\n' % current

324

current_file.write(current_str)

325

current_file.close()

326

# copy edit-log

327

if self.editlog is not None:

328

editlog_fname = opj(page_dir, 'edit-log')

329

self.editlog.copy(editlog_fname, self.renames, deleted=self.is_deleted)

330

# copy page revisions

331

if self.revisions is not None:

332

rev_dir = opj(page_dir, 'revisions')

333

os.makedirs(rev_dir)

334

for rev in self.revlist:

335

if create_rev:

336

self.revisions[rev].copy(rev_dir, self.renames)

337

else:

338

if int(rev) == self.current:

339

self.revisions[rev].copy(rev_dir, self.renames, convert=True)

340

else:

341

self.revisions[rev].copy(rev_dir, self.renames)

342

if create_rev and not self.is_deleted:

343

self.revisions[rev].copy(rev_dir, self.renames, convert=True, new_rev=rev+1)

344

345

# copy attachments

346

if self.attachments is not None:

347

attach_dir = opj(page_dir, 'attachments')

348

os.makedirs(attach_dir)

349

for fn, att in self.attachments.items():

350

# we have to check for renames here because we need the (old) pagename, too:

351

if ('FILE', self.name_old, fn) in self.renames:

352

fn_new = self.renames[('FILE', self.name_old, fn)]

353

if fn_new != fn:

354

print "Renaming file %r %r -> %r" % (self.name_old, fn, fn_new)

355

att.name = fn_new

356

att.copy(attach_dir)

357

358

def copy(self, pages_dir, renames):

359

self.renames = renames

360

self.read()

361

self.write(pages_dir)

362

363

364

class User:

365

""" represents a user with all related data """

366

def __init__(self, request, users_dir, uid):

367

self.request = request

368

self.uid = uid

369

self.users_dir = users_dir

370

self.profile = None

371

self.bookmarks = None

372

373

def read(self):

374

""" read profile and bookmarks data from disk """

375

self.profile = {}

376

fname = opj(self.users_dir, self.uid)

377

# read user profile

378

f = codecs.open(fname, 'r', config.charset)

379

for line in f:

380

line = line.replace(u'\r', '').replace(u'\n', '')

381

if not line.strip() or line.startswith(u'#'): # skip empty or comment lines

382

continue

383

try:

384

key, value = line.split(u'=', 1)

385

except Exception, err:

386

print "Error: User reader can not parse line %r from profile %r (%s)" % (line, fname, str(err))

387

continue

388

self.profile[key] = value

389

f.close()

390

# read bookmarks

391

self.bookmarks = {}

392

fname_pattern = opj(self.users_dir, "%s.*.bookmark" % self.uid)

393

for fname in glob.glob(fname_pattern):

394

f = file(fname, "r")

395

bookmark = f.read()

396

f.close()

397

wiki = fname.replace('.bookmark', '').replace(opj(self.users_dir, self.uid+'.'), '')

398

self.bookmarks[wiki] = int(bookmark)

399

# don't care about trail

400

401

def write(self, users_dir):

402

""" write profile and bookmarks data to disk """

403

fname = opj(users_dir, self.uid)

404

f = codecs.open(fname, 'w', config.charset)

405

for key, value in self.profile.items():

406

if key in (u'subscribed_pages', u'quicklinks'):

407

pages = value.split(u'\t')

408

for i in range(len(pages)):

409

pagename = pages[i]

410

try:

411

interwiki, pagename = pagename.split(u':', 1)

412

except:

413

interwiki, pagename = u'Self', pagename

414

if interwiki == u'Self' or interwiki == self.request.cfg.interwikiname:

415

if ('PAGE', pagename) in self.renames:

416

pagename = self.renames[('PAGE', pagename)]

417

pages[i] = u'%s:%s' % (interwiki, pagename)

418

key += '[]' # we have lists here

419

value = u'\t'.join(pages)

420

f.write(u"%s=%s\n" % (key, value))

421

else:

422

f.write(u"%s=%s\n" % (key, value))

423

bookmark_entries = [u'%s:%s' % item for item in self.bookmarks.items()]

424

key = u"bookmarks{}"

425

value = u'\t'.join(bookmark_entries)

426

f.write(u"%s=%s\n" % (key, value))

427

f.close()

428

# don't care about trail

429

430

def copy(self, users_dir, renames):

431

self.renames = renames

432

self.read()

433

self.write(users_dir)

434

435

436

class DataConverter(object):

437

def __init__(self, request, src_data_dir, dest_data_dir):

438

self.request = request

439

self.sdata = src_data_dir

440

self.ddata = dest_data_dir

441

self.pages = {}

442

self.users = {}

443

self.complete = {}

444

self.renames = {}

445

self.complete_fname = opj(self.sdata, 'complete.txt')

446

self.rename_fname1 = opj(self.sdata, 'rename1.txt')

447

self.rename_fname2 = opj(self.sdata, 'rename2.txt')

448

449

def pass1(self):

450

""" First create the rename list - the user has to review/edit it as

451

we can't decide about page/attachment names automatically.

452

"""

453

self.read_src()

454

# pages

455

for pn, p in self.pages.items():

456

p.read()

457

if not p.revisions:

458

continue # we don't care for pages with no revisions (trash)

459

if pn.endswith('/MoinEditorBackup'):

460

continue # we don't care for old editor backups

461

self.complete[('PAGE', pn)] = None

462

if "_" in pn:

463

# log all pagenames with underscores

464

self.renames[('PAGE', pn)] = None

465

if p.attachments is not None:

466

for fn in p.attachments:

467

try:

468

fn_str = fn.encode('ascii')

469

log = False # pure ascii filenames are no problem

470

except UnicodeEncodeError:

471

log = True # this file maybe has a strange representation in wiki markup

472

else:

473

if ' ' in fn_str or '%' in fn_str: # files with blanks need quoting

474

log = True

475

self.complete[('FILE', pn, fn)] = None

476

if log:

477

# log all strange attachment filenames

478

fn_str = fn.encode('utf-8')

479

self.renames[('FILE', pn, fn)] = None

480

self.save_list(self.complete_fname, self.complete)

481

self.save_list(self.rename_fname1, self.renames)

482

483

LIST_FIELDSEP = u'|' # in case | makes trouble, one can use \t tab char

484

485

def save_list(self, fname, what):

486

what_sorted = what.keys()

487

# make sure we have 3-tuples:

488

what_sorted = [(k + (None, ))[:3] for k in what_sorted]

489

# we only have python 2.3, thus no cmp keyword for the sort() call,

490

# thus we need to do it the more complicated way:

491

what_sorted = [(pn, fn, rtype) for rtype, pn, fn in what_sorted] # shuffle

492

what_sorted.sort() # sort

493

what_sorted = [(rtype, pn, fn) for pn, fn, rtype in what_sorted] # shuffle

494

f = codecs.open(fname, 'w', 'utf-8')

495

for rtype, pn, fn in what_sorted:

496

if rtype == 'PAGE':

497

line = (rtype, pn, pn)

498

elif rtype == 'FILE':

499

line = (rtype, pn, fn, fn)

500

line = self.LIST_FIELDSEP.join(line)

501

f.write(line + u'\n')

502

f.close()

503

504

def load_list(self, fname, what):

505

f = codecs.open(fname, 'r', 'utf-8')

506

for line in f:

507

line = line.rstrip()

508

if not line:

509

continue

510

t = line.split(self.LIST_FIELDSEP)

511

rtype, p1, p2, p3 = (t + [None]*3)[:4]

512

if rtype == u'PAGE':

513

what[(str(rtype), p1)] = p2

514

elif rtype == u'FILE':

515

what[(str(rtype), p1, p2)] = p3

516

f.close()

517

518

def pass2(self):

519

""" Second, read the (user edited) rename list and do the renamings everywhere. """

520

self.read_src()

521

#self.load_list(self.complete_fname, self.complete)

522

self.load_list(self.rename_fname2, self.renames)

523

self.write_dest()

524

525

def read_src(self):

526

# create Page objects in memory

527

pages_dir = opj(self.sdata, 'pages')

528

pagelist = listdir(pages_dir)

529

for qpagename in pagelist:

530

p = Page(self.request, pages_dir, qpagename)

531

self.pages[p.name] = p

532

533

# create User objects in memory

534

users_dir = opj(self.sdata, 'user')

535

user_re = re.compile(r'^\d+\.\d+(\.\d+)?$')

536

userlist = listdir(users_dir)

537

userlist = [f for f in userlist if user_re.match(f)]

538

for userid in userlist:

539

u = User(self.request, users_dir, userid)

540

self.users[u.uid] = u

541

542

# create log objects in memory

543

self.editlog = EditLog(self.request, opj(self.sdata, 'edit-log'))

544

self.eventlog = EventLog(self.request, opj(self.sdata, 'event-log'))

545

546

def write_dest(self):

547

self.init_dest()

548

# copy pages

549

pages_dir = opj(self.ddata, 'pages')

550

for pn, page in self.pages.items():

551

if pn.endswith('/MoinEditorBackup'):

552

continue # we don't care for old editor backups

553

page.copy(pages_dir, self.renames)

554

555

# copy users

556

users_dir = opj(self.ddata, 'user')

557

for user in self.users.values():

558

user.copy(users_dir, self.renames)

559

560

# copy logs

561

self.editlog.copy(opj(self.ddata, 'edit-log'), self.renames)

562

self.eventlog.copy(opj(self.ddata, 'event-log'), self.renames)

563

564

def init_dest(self):

565

try:

566

os.makedirs(self.ddata)

567

except:

568

pass

569

os.makedirs(opj(self.ddata, 'pages'))

570

os.makedirs(opj(self.ddata, 'user'))

571

copy_dir(opj(self.sdata, 'plugin'), opj(self.ddata, 'plugin'))

572

copy_file(opj(self.sdata, 'intermap.txt'), opj(self.ddata, 'intermap.txt'))

573

574

Older »