1
# -*- coding: iso-8859-1 -*-
3
MoinMoin - Wiki Utility Functions
5
@copyright: 2000-2004 Juergen Hermann <jh@web.de>,
8
2005-2008 MoinMoin:ThomasWaldmann,
9
2007 MoinMoin:ReimarBauer
10
@license: GNU GPL, see COPYING for details.
20
from MoinMoin import log
21
logging = log.getLogger(__name__)
23
from werkzeug.security import safe_str_cmp as safe_str_equal
25
from MoinMoin import config
26
from MoinMoin.support.python_compatibility import rsplit
27
from inspect import getargspec, isfunction, isclass, ismethod
29
from MoinMoin import web # needed so that next lines work:
31
from MoinMoin.util import pysupport, lock
34
class InvalidFileNameError(Exception):
35
""" Called when we find an invalid file name """
38
# constants for page names
40
PARENT_PREFIX_LEN = len(PARENT_PREFIX)
42
CHILD_PREFIX_LEN = len(CHILD_PREFIX)
44
#############################################################################
45
### Getting data from user/Sending data to user
46
#############################################################################
48
def decodeUnknownInput(text):
49
""" Decode unknown input, like text attachments
51
First we try utf-8 because it has special format, and it will decode
52
only utf-8 files. Then we try config.charset, then iso-8859-1 using
53
'replace'. We will never raise an exception, but may return junk
56
WARNING: Use this function only for data that you view, not for data
57
that you save in the wiki.
59
@param text: the text to decode, string
61
@return: decoded text (maybe wrong)
63
# Shortcut for unicode input
64
if isinstance(text, unicode):
68
return unicode(text, 'utf-8')
70
if config.charset not in ['utf-8', 'iso-8859-1']:
72
return unicode(text, config.charset)
75
return unicode(text, 'iso-8859-1', 'replace')
78
def decodeUserInput(s, charsets=[config.charset]):
80
Decodes input from the user.
82
@param s: the string to unquote
83
@param charsets: list of charsets to assume the string is in
85
@return: the unquoted string as unicode
87
for charset in charsets:
89
return s.decode(charset)
92
raise UnicodeError('The string %r cannot be decoded.' % s)
95
def url_quote(s, safe='/', want_unicode=None):
96
""" see werkzeug.url_quote, we use a different safe param default value """
98
assert want_unicode is None
99
except AssertionError:
100
log.exception("call with deprecated want_unicode param, please fix caller")
101
return werkzeug.url_quote(s, charset=config.charset, safe=safe)
103
def url_quote_plus(s, safe='/', want_unicode=None):
104
""" see werkzeug.url_quote_plus, we use a different safe param default value """
106
assert want_unicode is None
107
except AssertionError:
108
log.exception("call with deprecated want_unicode param, please fix caller")
109
return werkzeug.url_quote_plus(s, charset=config.charset, safe=safe)
111
def url_unquote(s, want_unicode=None):
112
""" see werkzeug.url_unquote """
114
assert want_unicode is None
115
except AssertionError:
116
log.exception("call with deprecated want_unicode param, please fix caller")
117
if isinstance(s, unicode):
118
s = s.encode(config.charset)
119
return werkzeug.url_unquote(s, charset=config.charset, errors='fallback:iso-8859-1')
122
def parseQueryString(qstr, want_unicode=None):
123
""" see werkzeug.url_decode
125
Please note: this returns a MultiDict, you might need to use dict() on
126
the result if your code expects a "normal" dict.
129
assert want_unicode is None
130
except AssertionError:
131
log.exception("call with deprecated want_unicode param, please fix caller")
132
return werkzeug.url_decode(qstr, charset=config.charset, errors='fallback:iso-8859-1',
133
decode_keys=False, include_empty=False)
135
def makeQueryString(qstr=None, want_unicode=None, **kw):
136
""" Make a querystring from arguments.
138
kw arguments overide values in qstr.
140
If a string is passed in, it's returned verbatim and keyword parameters are ignored.
142
See also: werkzeug.url_encode
144
@param qstr: dict to format as query string, using either ascii or unicode
145
@param kw: same as dict when using keywords, using ascii or unicode
147
@return: query string ready to use in a url
150
assert want_unicode is None
151
except AssertionError:
152
log.exception("call with deprecated want_unicode param, please fix caller")
155
elif isinstance(qstr, (str, unicode)):
157
if isinstance(qstr, dict):
159
return werkzeug.url_encode(qstr, charset=config.charset, encode_keys=True)
161
raise ValueError("Unsupported argument type, should be dict.")
164
def quoteWikinameURL(pagename, charset=config.charset):
165
""" Return a url encoding of filename in plain ascii
167
Use urllib.quote to quote any character that is not always safe.
169
@param pagename: the original pagename (unicode)
170
@param charset: url text encoding, 'utf-8' recommended. Other charset
171
might not be able to encode the page name and raise
172
UnicodeError. (default config.charset ('utf-8')).
174
@return: the quoted filename, all unsafe characters encoded
176
# XXX please note that urllib.quote and werkzeug.url_quote have
177
# XXX different defaults for safe=...
178
return werkzeug.url_quote(pagename, charset=charset, safe='/')
181
escape = werkzeug.escape
184
def clean_input(text, max_len=201):
186
replace CR, LF, TAB by whitespace
189
@param text: unicode text to clean (if we get str, we decode)
191
@return: cleaned text
193
# we only have input fields with max 200 chars, but spammers send us more
195
if length == 0 or length > max_len:
198
if isinstance(text, str):
199
# the translate() below can ONLY process unicode, thus, if we get
200
# str, we try to decode it using the usual coding:
201
text = text.decode(config.charset)
202
return text.translate(config.clean_input_translation_map)
205
def make_breakable(text, maxlen):
206
""" make a text breakable by inserting spaces into nonbreakable parts
208
text = text.split(" ")
211
if len(part) > maxlen:
213
newtext.append(part[:maxlen])
217
return " ".join(newtext)
219
########################################################################
221
########################################################################
223
# Precompiled patterns for file name [un]quoting
224
UNSAFE = re.compile(r'[^a-zA-Z0-9_]+')
225
QUOTED = re.compile(r'\(([a-fA-F0-9]+)\)')
228
def quoteWikinameFS(wikiname, charset=config.charset):
229
""" Return file system representation of a Unicode WikiName.
231
Warning: will raise UnicodeError if wikiname can not be encoded using
232
charset. The default value of config.charset, 'utf-8' can encode any
235
@param wikiname: Unicode string possibly containing non-ascii characters
236
@param charset: charset to encode string
238
@return: quoted name, safe for any file system
240
filename = wikiname.encode(charset)
244
for needle in UNSAFE.finditer(filename):
245
# append leading safe stuff
246
quoted.append(filename[location:needle.start()])
247
location = needle.end()
248
# Quote and append unsafe stuff
250
for character in needle.group():
251
quoted.append('%02x' % ord(character))
254
# append rest of string
255
quoted.append(filename[location:])
256
return ''.join(quoted)
259
def unquoteWikiname(filename, charsets=[config.charset]):
260
""" Return Unicode WikiName from quoted file name.
262
We raise an InvalidFileNameError if we find an invalid name, so the
263
wiki could alarm the admin or suggest the user to rename a page.
264
Invalid file names should never happen in normal use, but are rather
267
This function should be used only to unquote file names, not page
268
names we receive from the user. These are handled in request by
269
urllib.unquote, decodePagename and normalizePagename.
271
Todo: search clients of unquoteWikiname and check for exceptions.
273
@param filename: string using charset and possibly quoted parts
274
@param charsets: list of charsets used by string
275
@rtype: Unicode String
278
### Temporary fix start ###
279
# From some places we get called with Unicode strings
280
if isinstance(filename, type(u'')):
281
filename = filename.encode(config.charset)
282
### Temporary fix end ###
286
for needle in QUOTED.finditer(filename):
287
# append leading unquoted stuff
288
parts.append(filename[start:needle.start()])
290
# Append quoted stuff
291
group = needle.group(1)
292
# Filter invalid filenames
293
if (len(group) % 2 != 0):
294
raise InvalidFileNameError(filename)
296
for i in range(0, len(group), 2):
298
character = chr(int(byte, 16))
299
parts.append(character)
301
# byte not in hex, e.g 'xy'
302
raise InvalidFileNameError(filename)
304
# append rest of string
308
parts.append(filename[start:len(filename)])
309
wikiname = ''.join(parts)
311
# FIXME: This looks wrong, because at this stage "()" can be both errors
312
# like open "(" without close ")", or unquoted valid characters in the file name.
313
# Filter invalid filenames. Any left (xx) must be invalid
314
#if '(' in wikiname or ')' in wikiname:
315
# raise InvalidFileNameError(filename)
317
wikiname = decodeUserInput(wikiname, charsets)
321
def timestamp2version(ts):
322
""" Convert UNIX timestamp (may be float or int) to our version
324
We don't want to use floats, so we just scale by 1e6 to get
327
return long(ts*1000000L) # has to be long for py 2.2.x
329
def version2timestamp(v):
330
""" Convert version number to UNIX timestamp (float).
331
This must ONLY be used for display purposes.
336
# This is the list of meta attribute names to be treated as integers.
337
# IMPORTANT: do not use any meta attribute names with "-" (or any other chars
338
# invalid in python attribute names), use e.g. _ instead.
339
INTEGER_METAS = ['current', 'revision', # for page storage (moin 2.0)
340
'data_format_revision', # for data_dir format spec (use by mig scripts)
343
class MetaDict(dict):
344
""" store meta informations as a dict.
346
def __init__(self, metafilename, cache_directory):
347
""" create a MetaDict from metafilename """
349
self.metafilename = metafilename
351
lock_dir = os.path.join(cache_directory, '__metalock__')
352
self.rlock = lock.ReadLock(lock_dir, 60.0)
353
self.wlock = lock.WriteLock(lock_dir, 60.0)
355
if not self.rlock.acquire(3.0):
356
raise EnvironmentError("Could not lock in MetaDict")
363
""" get the meta dict from an arbitrary filename.
364
does not keep state, does uncached, direct disk access.
365
@param metafilename: the name of the file to read
366
@return: dict with all values or {} if empty or error
370
metafile = codecs.open(self.metafilename, "r", "utf-8")
371
meta = metafile.read() # this is much faster than the file's line-by-line iterator
375
for line in meta.splitlines():
376
key, value = line.split(':', 1)
377
value = value.strip()
378
if key in INTEGER_METAS:
380
dict.__setitem__(self, key, value)
383
""" put the meta dict into an arbitrary filename.
384
does not keep or modify state, does uncached, direct disk access.
385
@param metafilename: the name of the file to write
386
@param metadata: dict of the data to write to the file
389
for key, value in self.items():
390
if key in INTEGER_METAS:
392
meta.append("%s: %s" % (key, value))
393
meta = '\r\n'.join(meta)
395
metafile = codecs.open(self.metafilename, "w", "utf-8")
400
def sync(self, mtime_usecs=None):
401
""" No-Op except for that parameter """
402
if not mtime_usecs is None:
403
self.__setitem__('mtime', str(mtime_usecs))
406
def __getitem__(self, key):
407
""" We don't care for cache coherency here. """
408
return dict.__getitem__(self, key)
410
def __setitem__(self, key, value):
411
""" Sets a dictionary entry. """
412
if not self.wlock.acquire(5.0):
413
raise EnvironmentError("Could not lock in MetaDict")
415
self._get_meta() # refresh cache
417
oldvalue = dict.__getitem__(self, key)
420
if value != oldvalue:
421
dict.__setitem__(self, key, value)
422
self._put_meta() # sync cache
427
# Quoting of wiki names, file names, etc. (in the wiki markup) -----------------------------------
429
# don't ever change this - DEPRECATED, only needed for 1.5 > 1.6 migration conversion
433
#############################################################################
435
#############################################################################
436
INTERWIKI_PAGE = "InterWikiMap"
438
def generate_file_list(request):
439
""" generates a list of all files. for internal use. """
441
# order is important here, the local intermap file takes
442
# precedence over the shared one, and is thus read AFTER
444
intermap_files = request.cfg.shared_intermap
445
if not isinstance(intermap_files, list):
446
intermap_files = [intermap_files]
448
intermap_files = intermap_files[:]
449
intermap_files.append(os.path.join(request.cfg.data_dir, "intermap.txt"))
450
request.cfg.shared_intermap_files = [filename for filename in intermap_files
451
if filename and os.path.isfile(filename)]
454
def get_max_mtime(file_list, page):
455
""" Returns the highest modification time of the files in file_list and the
457
timestamps = [os.stat(filename).st_mtime for filename in file_list]
459
# exists() is cached and thus cheaper than mtime_usecs()
460
timestamps.append(version2timestamp(page.mtime_usecs()))
462
return max(timestamps)
464
return 0 # no files / pages there
466
def load_wikimap(request):
467
""" load interwiki map (once, and only on demand) """
468
from MoinMoin.Page import Page
470
now = int(time.time())
471
if getattr(request.cfg, "shared_intermap_files", None) is None:
472
generate_file_list(request)
475
_interwiki_list = request.cfg.cache.interwiki_list
476
old_mtime = request.cfg.cache.interwiki_mtime
477
if request.cfg.cache.interwiki_ts + (1*60) < now: # 1 minutes caching time
478
max_mtime = get_max_mtime(request.cfg.shared_intermap_files, Page(request, INTERWIKI_PAGE))
479
if max_mtime > old_mtime:
480
raise AttributeError # refresh cache
482
request.cfg.cache.interwiki_ts = now
483
except AttributeError:
487
for filename in request.cfg.shared_intermap_files:
488
f = codecs.open(filename, "r", config.charset)
489
lines.extend(f.readlines())
492
# add the contents of the InterWikiMap page
493
lines += Page(request, INTERWIKI_PAGE).get_raw_body().splitlines()
496
if not line or line[0] == '#':
499
line = "%s %s/InterWiki" % (line, request.script_root)
500
wikitag, urlprefix, dummy = line.split(None, 2)
504
_interwiki_list[wikitag] = urlprefix
508
# add own wiki as "Self" and by its configured name
509
_interwiki_list['Self'] = request.script_root + '/'
510
if request.cfg.interwikiname:
511
_interwiki_list[request.cfg.interwikiname] = request.script_root + '/'
514
request.cfg.cache.interwiki_list = _interwiki_list
515
request.cfg.cache.interwiki_ts = now
516
request.cfg.cache.interwiki_mtime = get_max_mtime(request.cfg.shared_intermap_files, Page(request, INTERWIKI_PAGE))
518
return _interwiki_list
520
def split_wiki(wikiurl):
524
*** DEPRECATED FUNCTION FOR OLD 1.5 SYNTAX - ONLY STILL HERE FOR THE 1.5 -> 1.6 MIGRATION ***
525
Use split_interwiki(), see below.
527
@param wikiurl: the url to split
531
# !!! use a regex here!
533
wikitag, tail = wikiurl.split(":", 1)
536
wikitag, tail = wikiurl.split("/", 1)
538
wikitag, tail = 'Self', wikiurl
541
def split_interwiki(wikiurl):
542
""" Split a interwiki name, into wikiname and pagename, e.g:
544
'MoinMoin:FrontPage' -> "MoinMoin", "FrontPage"
545
'FrontPage' -> "Self", "FrontPage"
546
'MoinMoin:Page with blanks' -> "MoinMoin", "Page with blanks"
547
'MoinMoin:' -> "MoinMoin", ""
549
can also be used for:
551
'attachment:filename with blanks.txt' -> "attachment", "filename with blanks.txt"
553
@param wikiurl: the url to split
555
@return: (wikiname, pagename)
558
wikiname, pagename = wikiurl.split(":", 1)
560
wikiname, pagename = 'Self', wikiurl
561
return wikiname, pagename
563
def resolve_wiki(request, wikiurl):
565
Resolve an interwiki link.
567
*** DEPRECATED FUNCTION FOR OLD 1.5 SYNTAX - ONLY STILL HERE FOR THE 1.5 -> 1.6 MIGRATION ***
568
Use resolve_interwiki(), see below.
570
@param request: the request object
571
@param wikiurl: the InterWiki:PageName link
573
@return: (wikitag, wikiurl, wikitail, err)
575
_interwiki_list = load_wikimap(request)
577
wikiname, pagename = split_wiki(wikiurl)
579
# return resolved url
580
if wikiname in _interwiki_list:
581
return (wikiname, _interwiki_list[wikiname], pagename, False)
583
return (wikiname, request.script_root, "/InterWiki", True)
585
def resolve_interwiki(request, wikiname, pagename):
586
""" Resolve an interwiki reference (wikiname:pagename).
588
@param request: the request object
589
@param wikiname: interwiki wiki name
590
@param pagename: interwiki page name
592
@return: (wikitag, wikiurl, wikitail, err)
594
_interwiki_list = load_wikimap(request)
595
if wikiname in _interwiki_list:
596
return (wikiname, _interwiki_list[wikiname], pagename, False)
598
return (wikiname, request.script_root, "/InterWiki", True)
600
def join_wiki(wikiurl, wikitail):
602
Add a (url_quoted) page name to an interwiki url.
604
Note: We can't know what kind of URL quoting a remote wiki expects.
605
We just use a utf-8 encoded string with standard URL quoting.
607
@param wikiurl: wiki url, maybe including a $PAGE placeholder
608
@param wikitail: page name
610
@return: generated URL of the page in the other wiki
612
wikitail = url_quote(wikitail)
613
if '$PAGE' in wikiurl:
614
return wikiurl.replace('$PAGE', wikitail)
616
return wikiurl + wikitail
619
#############################################################################
620
### Page types (based on page names)
621
#############################################################################
623
def isSystemPage(request, pagename):
624
""" Is this a system page?
626
@param request: the request object
627
@param pagename: the page name
629
@return: true if page is a system page
631
from MoinMoin import i18n
632
return pagename in i18n.system_pages or isTemplatePage(request, pagename)
635
def isTemplatePage(request, pagename):
636
""" Is this a template page?
638
@param pagename: the page name
640
@return: true if page is a template page
642
return request.cfg.cache.page_template_regexact.search(pagename) is not None
645
def isGroupPage(pagename, cfg):
646
""" Is this a name of group page?
648
@param pagename: the page name
650
@return: true if page is a form page
652
return cfg.cache.page_group_regexact.search(pagename) is not None
655
def filterCategoryPages(request, pagelist):
656
""" Return category pages in pagelist
658
WARNING: DO NOT USE THIS TO FILTER THE FULL PAGE LIST! Use
659
getPageList with a filter function.
661
If you pass a list with a single pagename, either that is returned
662
or an empty list, thus you can use this function like a `isCategoryPage`
665
@param pagelist: a list of pages
667
@return: only the category pages of pagelist
669
func = request.cfg.cache.page_category_regexact.search
670
return [pn for pn in pagelist if func(pn)]
673
def getLocalizedPage(request, pagename): # was: getSysPage
674
""" Get a system page according to user settings and available translations.
676
We include some special treatment for the case that <pagename> is the
677
currently rendered page, as this is the case for some pages used very
678
often, like FrontPage, RecentChanges etc. - in that case we reuse the
679
already existing page object instead creating a new one.
681
@param request: the request object
682
@param pagename: the name of the page
684
@return: the page object of that system page, using a translated page,
687
from MoinMoin.Page import Page
688
i18n_name = request.getText(pagename)
690
if i18n_name != pagename:
691
if request.page and i18n_name == request.page.page_name:
692
# do not create new object for current page
693
i18n_page = request.page
694
if i18n_page.exists():
697
i18n_page = Page(request, i18n_name)
698
if i18n_page.exists():
701
# if we failed getting a translated version of <pagename>,
702
# we fall back to english
704
if request.page and pagename == request.page.page_name:
705
# do not create new object for current page
706
pageobj = request.page
708
pageobj = Page(request, pagename)
712
def getFrontPage(request):
713
""" Convenience function to get localized front page
715
@param request: current request
717
@return localized page_front_page, if there is a translation
719
return getLocalizedPage(request, request.cfg.page_front_page)
722
def getHomePage(request, username=None):
724
Get a user's homepage, or return None for anon users and
725
those who have not created a homepage.
727
DEPRECATED - try to use getInterwikiHomePage (see below)
729
@param request: the request object
730
@param username: the user's name
732
@return: user's homepage object - or None
734
from MoinMoin.Page import Page
735
# default to current user
736
if username is None and request.user.valid:
737
username = request.user.name
742
page = Page(request, username)
749
def getInterwikiHomePage(request, username=None):
751
Get a user's homepage.
753
cfg.user_homewiki influences behaviour of this:
754
'Self' does mean we store user homepage in THIS wiki.
755
When set to our own interwikiname, it behaves like with 'Self'.
757
'SomeOtherWiki' means we store user homepages in another wiki.
759
@param request: the request object
760
@param username: the user's name
761
@rtype: tuple (or None for anon users)
762
@return: (wikiname, pagename)
764
# default to current user
765
if username is None and request.user.valid:
766
username = request.user.name
768
return None # anon user
770
homewiki = request.cfg.user_homewiki
771
if homewiki == request.cfg.interwikiname:
774
return homewiki, username
777
def AbsPageName(context, pagename):
779
Return the absolute pagename for a (possibly) relative pagename.
781
@param context: name of the page where "pagename" appears on
782
@param pagename: the (possibly relative) page name
784
@return: the absolute page name
786
if pagename.startswith(PARENT_PREFIX):
787
while context and pagename.startswith(PARENT_PREFIX):
788
context = '/'.join(context.split('/')[:-1])
789
pagename = pagename[PARENT_PREFIX_LEN:]
790
pagename = '/'.join(filter(None, [context, pagename, ]))
791
elif pagename.startswith(CHILD_PREFIX):
793
pagename = context + '/' + pagename[CHILD_PREFIX_LEN:]
795
pagename = pagename[CHILD_PREFIX_LEN:]
798
def RelPageName(context, pagename):
800
Return the relative pagename for some context.
802
@param context: name of the page where "pagename" appears on
803
@param pagename: the absolute page name
805
@return: the relative page name
808
# special case, context is some "virtual root" page with name == ''
809
# every page is a subpage of this virtual root
810
return CHILD_PREFIX + pagename
811
elif pagename.startswith(context + CHILD_PREFIX):
813
return pagename[len(context):]
815
# some kind of sister/aunt
816
context_frags = context.split('/') # A, B, C, D, E
817
pagename_frags = pagename.split('/') # A, B, C, F
818
# first throw away common parents:
820
for cf, pf in zip(context_frags, pagename_frags):
825
context_frags = context_frags[common:] # D, E
826
pagename_frags = pagename_frags[common:] # F
827
go_up = len(context_frags)
828
return PARENT_PREFIX * go_up + '/'.join(pagename_frags)
831
def pagelinkmarkup(pagename, text=None):
832
""" return markup that can be used as link to page <pagename> """
833
from MoinMoin.parser.text_moin_wiki import Parser
834
if re.match(Parser.word_rule + "$", pagename, re.U|re.X) and \
835
(text is None or text == pagename):
838
if text is None or text == pagename:
842
return u'[[%s%s]]' % (pagename, text)
844
#############################################################################
846
#############################################################################
850
# OpenOffice 2.x & other open document stuff
851
'.odt': 'application/vnd.oasis.opendocument.text',
852
'.ods': 'application/vnd.oasis.opendocument.spreadsheet',
853
'.odp': 'application/vnd.oasis.opendocument.presentation',
854
'.odg': 'application/vnd.oasis.opendocument.graphics',
855
'.odc': 'application/vnd.oasis.opendocument.chart',
856
'.odf': 'application/vnd.oasis.opendocument.formula',
857
'.odb': 'application/vnd.oasis.opendocument.database',
858
'.odi': 'application/vnd.oasis.opendocument.image',
859
'.odm': 'application/vnd.oasis.opendocument.text-master',
860
'.ott': 'application/vnd.oasis.opendocument.text-template',
861
'.ots': 'application/vnd.oasis.opendocument.spreadsheet-template',
862
'.otp': 'application/vnd.oasis.opendocument.presentation-template',
863
'.otg': 'application/vnd.oasis.opendocument.graphics-template',
864
# some systems (like Mac OS X) don't have some of these:
865
'.patch': 'text/x-diff',
866
'.diff': 'text/x-diff',
867
'.py': 'text/x-python',
868
'.cfg': 'text/plain',
869
'.conf': 'text/plain',
870
'.irc': 'text/plain',
871
'.md5': 'text/plain',
873
'.flv': 'video/x-flv',
874
'.wmv': 'video/x-ms-wmv',
875
'.swf': 'application/x-shockwave-flash',
876
'.moin': 'text/moin-wiki',
877
'.creole': 'text/creole',
878
# Windows Server 2003 / Python 2.7 has no or strange entries for these:
879
'.svg': 'image/svg+xml',
880
'.svgz': 'image/svg+xml',
882
'.jpg': 'image/jpeg',
883
'.jpeg': 'image/jpeg',
887
# add all mimetype patterns of pygments
888
import pygments.lexers
890
for name, short, patterns, mime in pygments.lexers.get_all_lexers():
891
for pattern in patterns:
892
if pattern.startswith('*.') and mime:
893
MIMETYPES_MORE[pattern[1:]] = mime[0]
895
[mimetypes.add_type(mimetype, ext, True) for ext, mimetype in MIMETYPES_MORE.items()]
897
MIMETYPES_sanitize_mapping = {
898
# this stuff is text, but got application/* for unknown reasons
899
('application', 'docbook+xml'): ('text', 'docbook'),
900
('application', 'x-latex'): ('text', 'latex'),
901
('application', 'x-tex'): ('text', 'tex'),
902
('application', 'javascript'): ('text', 'javascript'),
905
MIMETYPES_spoil_mapping = {} # inverse mapping of above
906
for _key, _value in MIMETYPES_sanitize_mapping.items():
907
MIMETYPES_spoil_mapping[_value] = _key
910
class MimeType(object):
911
""" represents a mimetype like text/plain """
913
def __init__(self, mimestr=None, filename=None):
914
self.major = self.minor = None # sanitized mime type and subtype
915
self.params = {} # parameters like "charset" or others
916
self.charset = None # this stays None until we know for sure!
917
self.raw_mimestr = mimestr
920
self.parse_mimetype(mimestr)
922
self.parse_filename(filename)
924
def parse_filename(self, filename):
925
mtype, encoding = mimetypes.guess_type(filename)
927
mtype = 'application/octet-stream'
928
self.parse_mimetype(mtype)
930
def parse_mimetype(self, mimestr):
931
""" take a string like used in content-type and parse it into components,
932
alternatively it also can process some abbreviated string like "wiki"
934
parameters = mimestr.split(";")
935
parameters = [p.strip() for p in parameters]
936
mimetype, parameters = parameters[0], parameters[1:]
937
mimetype = mimetype.split('/')
938
if len(mimetype) >= 2:
939
major, minor = mimetype[:2] # we just ignore more than 2 parts
941
major, minor = self.parse_format(mimetype[0])
942
self.major = major.lower()
943
self.minor = minor.lower()
944
for param in parameters:
945
key, value = param.split('=')
946
if value[0] == '"' and value[-1] == '"': # remove quotes
948
self.params[key.lower()] = value
949
if 'charset' in self.params:
950
self.charset = self.params['charset'].lower()
953
def parse_format(self, format):
954
""" maps from what we currently use on-page in a #format xxx processing
955
instruction to a sanitized mimetype major, minor tuple.
956
can also be user later for easier entry by the user, so he can just
957
type "wiki" instead of "text/moin-wiki".
959
format = format.lower()
960
if format in config.parser_text_mimetype:
961
mimetype = 'text', format
964
'wiki': ('text', 'moin-wiki'),
965
'irc': ('text', 'irssi'),
968
mimetype = mapping[format]
970
mimetype = 'text', 'x-%s' % format
974
""" convert to some representation that makes sense - this is not necessarily
975
conformant to /etc/mime.types or IANA listing, but if something is
976
readable text, we will return some text/* mimetype, not application/*,
977
because we need text/plain as fallback and not application/octet-stream.
979
self.major, self.minor = MIMETYPES_sanitize_mapping.get((self.major, self.minor), (self.major, self.minor))
982
""" this returns something conformant to /etc/mime.type or IANA as a string,
983
kind of inverse operation of sanitize(), but doesn't change self
985
major, minor = MIMETYPES_spoil_mapping.get((self.major, self.minor), (self.major, self.minor))
986
return self.content_type(major, minor)
988
def content_type(self, major=None, minor=None, charset=None, params=None):
989
""" return a string suitable for Content-Type header
991
major = major or self.major
992
minor = minor or self.minor
993
params = params or self.params or {}
995
charset = charset or self.charset or params.get('charset', config.charset)
996
params['charset'] = charset
997
mimestr = "%s/%s" % (major, minor)
998
params = ['%s="%s"' % (key.lower(), value) for key, value in params.items()]
999
params.insert(0, mimestr)
1000
return "; ".join(params)
1002
def mime_type(self):
1003
""" return a string major/minor only, no params """
1004
return "%s/%s" % (self.major, self.minor)
1006
def module_name(self):
1007
""" convert this mimetype to a string useable as python module name,
1008
we yield the exact module name first and then proceed to shorter
1009
module names (useful for falling back to them, if the more special
1010
module is not found) - e.g. first "text_python", next "text".
1011
Finally, we yield "application_octet_stream" as the most general
1013
Hint: the fallback handler module for text/* should be implemented
1014
in module "text" (not "text_plain")
1016
mimetype = self.mime_type()
1017
modname = mimetype.replace("/", "_").replace("-", "_").replace(".", "_")
1018
fragments = modname.split('_')
1019
for length in range(len(fragments), 1, -1):
1020
yield "_".join(fragments[:length])
1021
yield self.raw_mimestr
1023
yield "application_octet_stream"
1026
#############################################################################
1028
#############################################################################
1030
class PluginError(Exception):
1031
""" Base class for plugin errors """
1033
class PluginMissingError(PluginError):
1034
""" Raised when a plugin is not found """
1036
class PluginAttributeError(PluginError):
1037
""" Raised when plugin does not contain an attribtue """
1040
def importPlugin(cfg, kind, name, function="execute"):
1041
""" Import wiki or builtin plugin
1043
Returns <function> attr from a plugin module <name>.
1044
If <function> attr is missing, raise PluginAttributeError.
1045
If <function> is None, return the whole module object.
1047
If <name> plugin can not be imported, raise PluginMissingError.
1049
kind may be one of 'action', 'formatter', 'macro', 'parser' or any other
1050
directory that exist in MoinMoin or data/plugin.
1052
Wiki plugins will always override builtin plugins. If you want
1053
specific plugin, use either importWikiPlugin or importBuiltinPlugin
1056
@param cfg: wiki config instance
1057
@param kind: what kind of module we want to import
1058
@param name: the name of the module
1059
@param function: the function name
1061
@return: "function" of module "name" of kind "kind", or None
1064
return importWikiPlugin(cfg, kind, name, function)
1065
except PluginMissingError:
1066
return importBuiltinPlugin(kind, name, function)
1069
def importWikiPlugin(cfg, kind, name, function="execute"):
1070
""" Import plugin from the wiki data directory
1072
See importPlugin docstring.
1074
plugins = wikiPlugins(kind, cfg)
1075
modname = plugins.get(name, None)
1077
raise PluginMissingError()
1078
moduleName = '%s.%s' % (modname, name)
1079
return importNameFromPlugin(moduleName, function)
1082
def importBuiltinPlugin(kind, name, function="execute"):
1083
""" Import builtin plugin from MoinMoin package
1085
See importPlugin docstring.
1087
if not name in builtinPlugins(kind):
1088
raise PluginMissingError()
1089
moduleName = 'MoinMoin.%s.%s' % (kind, name)
1090
return importNameFromPlugin(moduleName, function)
1093
def importNameFromPlugin(moduleName, name):
1094
""" Return <name> attr from <moduleName> module,
1095
raise PluginAttributeError if name does not exist.
1097
If name is None, return the <moduleName> module object.
1103
module = __import__(moduleName, globals(), {}, fromlist)
1105
# module has the obj for module <moduleName>
1107
return getattr(module, name)
1108
except AttributeError:
1109
raise PluginAttributeError
1111
# module now has the toplevel module of <moduleName> (see __import__ docs!)
1112
components = moduleName.split('.')
1113
for comp in components[1:]:
1114
module = getattr(module, comp)
1118
def builtinPlugins(kind):
1119
""" Gets a list of modules in MoinMoin.'kind'
1121
@param kind: what kind of modules we look for
1123
@return: module names
1125
modulename = "MoinMoin." + kind
1126
return pysupport.importName(modulename, "modules")
1129
def wikiPlugins(kind, cfg):
1131
Gets a dict containing the names of all plugins of @kind
1132
as the key and the containing module name as the value.
1134
@param kind: what kind of modules we look for
1136
@return: plugin name to containing module name mapping
1138
# short-cut if we've loaded the dict already
1139
# (or already failed to load it)
1140
cache = cfg._site_plugin_lists
1142
result = cache[kind]
1145
for modname in cfg._plugin_modules:
1147
module = pysupport.importName(modname, kind)
1148
packagepath = os.path.dirname(module.__file__)
1149
plugins = pysupport.getPluginModules(packagepath)
1152
result[p] = '%s.%s' % (modname, kind)
1153
except AttributeError:
1155
cache[kind] = result
1159
def getPlugins(kind, cfg):
1160
""" Gets a list of plugin names of kind
1162
@param kind: what kind of modules we look for
1164
@return: module names
1166
# Copy names from builtin plugins - so we dont destroy the value
1167
all_plugins = builtinPlugins(kind)[:]
1169
# Add extension plugins without duplicates
1170
for plugin in wikiPlugins(kind, cfg):
1171
if plugin not in all_plugins:
1172
all_plugins.append(plugin)
1177
def searchAndImportPlugin(cfg, type, name, what=None):
1178
type2classname = {"parser": "Parser",
1179
"formatter": "Formatter",
1182
what = type2classname[type]
1185
for module_name in mt.module_name():
1187
plugin = importPlugin(cfg, type, module_name, what)
1189
except PluginMissingError:
1192
raise PluginMissingError("Plugin not found! (%r %r %r)" % (type, name, what))
1196
#############################################################################
1198
#############################################################################
1200
def getParserForExtension(cfg, extension):
1202
Returns the Parser class of the parser fit to handle a file
1203
with the given extension. The extension should be in the same
1204
format as os.path.splitext returns it (i.e. with the dot).
1205
Returns None if no parser willing to handle is found.
1206
The dict of extensions is cached in the config object.
1208
@param cfg: the Config instance for the wiki in question
1209
@param extension: the filename extension including the dot
1211
@returns: the parser class or None
1213
if not hasattr(cfg.cache, 'EXT_TO_PARSER'):
1215
parser_plugins = getPlugins('parser', cfg)
1216
# force the 'highlight' parser to be the first entry in the list
1217
# this makes it possible to overwrite some mapping entries later, so that
1218
# moin will use some "better" parser for some filename extensions
1219
parser_plugins.remove('highlight')
1220
parser_plugins = ['highlight'] + parser_plugins
1221
for pname in parser_plugins:
1223
Parser = importPlugin(cfg, 'parser', pname, 'Parser')
1224
except PluginMissingError:
1226
if hasattr(Parser, 'extensions'):
1227
exts = Parser.extensions
1228
if isinstance(exts, list):
1231
elif str(exts) == '*':
1233
cfg.cache.EXT_TO_PARSER = etp
1234
cfg.cache.EXT_TO_PARSER_DEFAULT = etd
1236
return cfg.cache.EXT_TO_PARSER.get(extension, cfg.cache.EXT_TO_PARSER_DEFAULT)
1239
#############################################################################
1240
### Parameter parsing
1241
#############################################################################
1243
class BracketError(Exception):
1246
class BracketUnexpectedCloseError(BracketError):
1247
def __init__(self, bracket):
1248
self.bracket = bracket
1249
BracketError.__init__(self, "Unexpected closing bracket %s" % bracket)
1251
class BracketMissingCloseError(BracketError):
1252
def __init__(self, bracket):
1253
self.bracket = bracket
1254
BracketError.__init__(self, "Missing closing bracket %s" % bracket)
1258
Trivial container-class holding a single character for
1259
the possible prefixes for parse_quoted_separated_ext
1260
and implementing rich equal comparison.
1262
def __init__(self, prefix):
1263
self.prefix = prefix
1265
def __eq__(self, other):
1266
return isinstance(other, ParserPrefix) and other.prefix == self.prefix
1269
return '<ParserPrefix(%s)>' % self.prefix.encode('utf-8')
1271
def parse_quoted_separated_ext(args, separator=None, name_value_separator=None,
1272
brackets=None, seplimit=0, multikey=False,
1273
prefixes=None, quotes='"'):
1275
Parses the given string according to the other parameters.
1277
Items can be quoted with any character from the quotes parameter
1278
and each quote can be escaped by doubling it, the separator and
1279
name_value_separator can both be quoted, when name_value_separator
1280
is set then the name can also be quoted.
1282
Values that are not given are returned as None, while the
1283
empty string as a value can be achieved by quoting it.
1285
If a name or value does not start with a quote, then the quote
1286
looses its special meaning for that name or value, unless it
1287
starts with one of the given prefixes (the parameter is unicode
1288
containing all allowed prefixes.) The prefixes will be returned
1289
as ParserPrefix() instances in the first element of the tuple
1290
for that particular argument.
1292
If multiple separators follow each other, this is treated as
1293
having None arguments inbetween, that is also true for when
1294
space is used as separators (when separator is None), filter
1295
them out afterwards.
1297
The function can also do bracketing, i.e. parse expressions
1298
that contain things like
1299
"(a (a b))" to ['(', 'a', ['(', 'a', 'b']],
1300
in this case, as in this example, the returned list will
1301
contain sub-lists and the brackets parameter must be a list
1302
of opening and closing brackets, e.g.
1303
brackets = ['()', '<>']
1304
Each sub-list's first item is the opening bracket used for
1306
Nesting will be observed between the different types of
1307
brackets given. If bracketing doesn't match, a BracketError
1308
instance is raised with a 'bracket' property indicating the
1309
type of missing or unexpected bracket, the instance will be
1310
either of the class BracketMissingCloseError or of the class
1311
BracketUnexpectedCloseError.
1313
If multikey is True (along with setting name_value_separator),
1314
then the returned tuples for (key, value) pairs can also have
1316
"a=b=c" -> ('a', 'b', 'c')
1318
@param args: arguments to parse
1319
@param separator: the argument separator, defaults to None, meaning any
1320
space separates arguments
1321
@param name_value_separator: separator for name=value, default '=',
1322
name=value keywords not parsed if evaluates to False
1323
@param brackets: a list of two-character strings giving
1324
opening and closing brackets
1325
@param seplimit: limits the number of parsed arguments
1326
@param multikey: multiple keys allowed for a single value
1328
@returns: list of unicode strings and tuples containing
1329
unicode strings, or lists containing the same for
1333
assert name_value_separator is None or name_value_separator != separator
1334
assert name_value_separator is None or len(name_value_separator) == 1
1335
if not isinstance(args, unicode):
1336
raise TypeError('args must be unicode')
1338
result = [] # result list
1339
cur = [None] # current item
1340
quoted = None # we're inside quotes, indicates quote character used
1341
skipquote = 0 # next quote is a quoted quote
1342
noquote = False # no quotes expected because word didn't start with one
1343
seplimit_reached = False # number of separators exhausted
1344
separator_count = 0 # number of separators encountered
1345
SPACE = [' ', '\t', ]
1346
nextitemsep = [separator] # used for skipping trailing space
1347
SPACE = [' ', '\t', ]
1348
if separator is None:
1349
nextitemsep = SPACE[:]
1352
nextitemsep = [separator] # used for skipping trailing space
1353
separators = [separator]
1354
if name_value_separator:
1355
nextitemsep.append(name_value_separator)
1357
# bracketing support
1361
matchingbracket = {}
1363
for o, c in brackets:
1364
assert not o in opening
1366
assert not c in closing
1368
matchingbracket[o] = c
1370
def additem(result, cur, separator_count, nextitemsep):
1374
result.append(tuple(cur))
1377
separator_count += 1
1378
seplimit_reached = False
1379
if seplimit and separator_count >= seplimit:
1380
seplimit_reached = True
1381
nextitemsep = [n for n in nextitemsep if n in separators]
1383
return cur, noquote, separator_count, seplimit_reached, nextitemsep
1392
if not separator is None and not quoted and char in SPACE:
1394
# accumulate all space
1395
while char in SPACE and idx < max - 1:
1399
# remove space if args end with it
1400
if char in SPACE and idx == max - 1:
1402
# remove space at end of argument
1403
if char in nextitemsep:
1406
if len(cur) and cur[-1]:
1407
cur[-1] = cur[-1] + spaces
1408
elif not quoted and char == name_value_separator:
1409
if multikey or len(cur) == 1:
1415
cur[-1] += name_value_separator
1419
elif not quoted and not seplimit_reached and char in separators:
1420
(cur, noquote, separator_count, seplimit_reached,
1421
nextitemsep) = additem(result, cur, separator_count, nextitemsep)
1422
elif not quoted and not noquote and char in quotes:
1423
if len(cur) and cur[-1] is None:
1427
elif char == quoted and not skipquote:
1429
skipquote = 2 # will be decremented right away
1432
elif not quoted and char in opening:
1433
while len(cur) and cur[-1] is None:
1435
(cur, noquote, separator_count, seplimit_reached,
1436
nextitemsep) = additem(result, cur, separator_count, nextitemsep)
1437
bracketstack.append((matchingbracket[char], result))
1439
elif not quoted and char in closing:
1440
while len(cur) and cur[-1] is None:
1442
(cur, noquote, separator_count, seplimit_reached,
1443
nextitemsep) = additem(result, cur, separator_count, nextitemsep)
1445
if not bracketstack:
1446
raise BracketUnexpectedCloseError(char)
1447
expected, oldresult = bracketstack[-1]
1448
if not expected == char:
1449
raise BracketUnexpectedCloseError(char)
1450
del bracketstack[-1]
1451
oldresult.append(result)
1453
elif not quoted and prefixes and char in prefixes and cur == [None]:
1454
cur = [ParserPrefix(char)]
1469
raise BracketMissingCloseError(bracketstack[-1][0])
1476
cur[-1] = quoted + cur[-1]
1480
additem(result, cur, separator_count, nextitemsep)
1484
def parse_quoted_separated(args, separator=',', name_value=True, seplimit=0):
1488
name_value_separator = '='
1492
name_value_separator = None
1494
l = parse_quoted_separated_ext(args, separator=separator,
1495
name_value_separator=name_value_separator,
1498
if isinstance(item, tuple):
1502
keywords[key] = value
1503
positional = trailing
1505
positional.append(item)
1508
return result, keywords, trailing
1511
def get_bool(request, arg, name=None, default=None):
1513
For use with values returned from parse_quoted_separated or given
1514
as macro parameters, return a boolean from a unicode string.
1515
Valid input is 'true'/'false', 'yes'/'no' and '1'/'0' or None for
1518
@param request: A request instance
1519
@param arg: The argument, may be None or a unicode string
1520
@param name: Name of the argument, for error messages
1521
@param default: default value if arg is None
1522
@rtype: boolean or None
1523
@returns: the boolean value of the string according to above rules
1527
assert default is None or isinstance(default, bool)
1530
elif not isinstance(arg, unicode):
1531
raise TypeError('Argument must be None or unicode')
1533
if arg in [u'0', u'false', u'no']:
1535
elif arg in [u'1', u'true', u'yes']:
1540
_('Argument "%s" must be a boolean value, not "%s"') % (
1544
_('Argument must be a boolean value, not "%s"') % arg)
1547
def get_int(request, arg, name=None, default=None):
1549
For use with values returned from parse_quoted_separated or given
1550
as macro parameters, return an integer from a unicode string
1551
containing the decimal representation of a number.
1552
None is a valid input and yields the default value.
1554
@param request: A request instance
1555
@param arg: The argument, may be None or a unicode string
1556
@param name: Name of the argument, for error messages
1557
@param default: default value if arg is None
1559
@returns: the integer value of the string (or default value)
1562
assert default is None or isinstance(default, (int, long))
1565
elif not isinstance(arg, unicode):
1566
raise TypeError('Argument must be None or unicode')
1572
_('Argument "%s" must be an integer value, not "%s"') % (
1576
_('Argument must be an integer value, not "%s"') % arg)
1579
def get_float(request, arg, name=None, default=None):
1581
For use with values returned from parse_quoted_separated or given
1582
as macro parameters, return a float from a unicode string.
1583
None is a valid input and yields the default value.
1585
@param request: A request instance
1586
@param arg: The argument, may be None or a unicode string
1587
@param name: Name of the argument, for error messages
1588
@param default: default return value if arg is None
1589
@rtype: float or None
1590
@returns: the float value of the string (or default value)
1593
assert default is None or isinstance(default, (int, long, float))
1596
elif not isinstance(arg, unicode):
1597
raise TypeError('Argument must be None or unicode')
1603
_('Argument "%s" must be a floating point value, not "%s"') % (
1607
_('Argument must be a floating point value, not "%s"') % arg)
1610
def get_complex(request, arg, name=None, default=None):
1612
For use with values returned from parse_quoted_separated or given
1613
as macro parameters, return a complex from a unicode string.
1614
None is a valid input and yields the default value.
1616
@param request: A request instance
1617
@param arg: The argument, may be None or a unicode string
1618
@param name: Name of the argument, for error messages
1619
@param default: default return value if arg is None
1620
@rtype: complex or None
1621
@returns: the complex value of the string (or default value)
1624
assert default is None or isinstance(default, (int, long, float, complex))
1627
elif not isinstance(arg, unicode):
1628
raise TypeError('Argument must be None or unicode')
1630
# allow writing 'i' instead of 'j'
1631
arg = arg.replace('i', 'j').replace('I', 'j')
1636
_('Argument "%s" must be a complex value, not "%s"') % (
1640
_('Argument must be a complex value, not "%s"') % arg)
1643
def get_unicode(request, arg, name=None, default=None):
1645
For use with values returned from parse_quoted_separated or given
1646
as macro parameters, return a unicode string from a unicode string.
1647
None is a valid input and yields the default value.
1649
@param request: A request instance
1650
@param arg: The argument, may be None or a unicode string
1651
@param name: Name of the argument, for error messages
1652
@param default: default return value if arg is None;
1653
@rtype: unicode or None
1654
@returns: the unicode string (or default value)
1656
assert default is None or isinstance(default, unicode)
1659
elif not isinstance(arg, unicode):
1660
raise TypeError('Argument must be None or unicode')
1665
def get_choice(request, arg, name=None, choices=[None], default_none=False):
1667
For use with values returned from parse_quoted_separated or given
1668
as macro parameters, return a unicode string that must be in the
1669
choices given. None is a valid input and yields first of the valid
1672
@param request: A request instance
1673
@param arg: The argument, may be None or a unicode string
1674
@param name: Name of the argument, for error messages
1675
@param choices: the possible choices
1676
@param default_none: If False (default), get_choice returns first available
1677
choice if arg is None. If True, get_choice returns
1678
None if arg is None. This is useful if some arg value
1679
is required (no default choice).
1680
@rtype: unicode or None
1681
@returns: the unicode string (or default value)
1683
assert isinstance(choices, (tuple, list))
1689
elif not isinstance(arg, unicode):
1690
raise TypeError('Argument must be None or unicode')
1691
elif not arg in choices:
1695
_('Argument "%s" must be one of "%s", not "%s"') % (
1696
name, '", "'.join([repr(choice) for choice in choices]),
1700
_('Argument must be one of "%s", not "%s"') % (
1701
'", "'.join([repr(choice) for choice in choices]), arg))
1708
Base class for new argument parsers for
1709
invoke_extension_function.
1714
def parse_argument(self, s):
1716
Parse the argument given in s (a string) and return
1717
the argument for the extension function.
1719
raise NotImplementedError
1721
def get_default(self):
1723
Return the default for this argument.
1725
raise NotImplementedError
1728
class UnitArgument(IEFArgument):
1730
Argument class for invoke_extension_function that forces
1731
having any of the specified units given for a value.
1733
Note that the default unit is "mm".
1735
Use, for example, "UnitArgument('7mm', float, ['%', 'mm'])".
1737
If the defaultunit parameter is given, any argument that
1738
can be converted into the given argtype is assumed to have
1739
the default unit. NOTE: This doesn't work with a choice
1740
(tuple or list) argtype.
1742
def __init__(self, default, argtype, units=['mm'], defaultunit=None):
1744
Initialise a UnitArgument giving the default,
1745
argument type and the permitted units.
1747
IEFArgument.__init__(self)
1748
self._units = list(units)
1749
self._units.sort(lambda x, y: len(y) - len(x))
1750
self._type = argtype
1751
self._defaultunit = defaultunit
1752
assert defaultunit is None or defaultunit in units
1753
if default is not None:
1754
self._default = self.parse_argument(default)
1756
self._default = None
1758
def parse_argument(self, s):
1759
for unit in self._units:
1760
if s.endswith(unit):
1761
ret = (self._type(s[:len(s) - len(unit)]), unit)
1763
if self._defaultunit is not None:
1765
return (self._type(s), self._defaultunit)
1768
units = ', '.join(self._units)
1769
## XXX: how can we translate this?
1770
raise ValueError("Invalid unit in value %s (allowed units: %s)" % (s, units))
1772
def get_default(self):
1773
return self._default
1778
Wrap a type in this class and give it as default argument
1779
for a function passed to invoke_extension_function() in
1780
order to get generic checking that the argument is given.
1782
def __init__(self, argtype):
1784
Initialise a required_arg
1785
@param argtype: the type the argument should have
1787
if not (argtype in (bool, int, long, float, complex, unicode) or
1788
isinstance(argtype, (IEFArgument, tuple, list))):
1789
raise TypeError("argtype must be a valid type")
1790
self.argtype = argtype
1793
def invoke_extension_function(request, function, args, fixed_args=[]):
1795
Parses arguments for an extension call and calls the extension
1796
function with the arguments.
1798
If the macro function has a default value that is a bool,
1799
int, long, float or unicode object, then the given value
1800
is converted to the type of that default value before passing
1801
it to the macro function. That way, macros need not call the
1802
wikiutil.get_* functions for any arguments that have a default.
1804
@param request: the request object
1805
@param function: the function to invoke
1806
@param args: unicode string with arguments (or evaluating to False)
1807
@param fixed_args: fixed arguments to pass as the first arguments
1808
@returns: the return value from the function called
1811
def _convert_arg(request, value, default, name=None):
1813
Using the get_* functions, convert argument to the type of the default
1814
if that is any of bool, int, long, float or unicode; if the default
1815
is the type itself then convert to that type (keeps None) or if the
1816
default is a list require one of the list items.
1818
In other cases return the value itself.
1820
# if extending this, extend required_arg as well!
1821
if isinstance(default, bool):
1822
return get_bool(request, value, name, default)
1823
elif isinstance(default, (int, long)):
1824
return get_int(request, value, name, default)
1825
elif isinstance(default, float):
1826
return get_float(request, value, name, default)
1827
elif isinstance(default, complex):
1828
return get_complex(request, value, name, default)
1829
elif isinstance(default, unicode):
1830
return get_unicode(request, value, name, default)
1831
elif isinstance(default, (tuple, list)):
1832
return get_choice(request, value, name, default)
1833
elif default is bool:
1834
return get_bool(request, value, name)
1835
elif default is int or default is long:
1836
return get_int(request, value, name)
1837
elif default is float:
1838
return get_float(request, value, name)
1839
elif default is complex:
1840
return get_complex(request, value, name)
1841
elif isinstance(default, IEFArgument):
1842
# defaults handled later
1845
return default.parse_argument(value)
1846
elif isinstance(default, required_arg):
1847
if isinstance(default.argtype, (tuple, list)):
1848
# treat choice specially and return None if no choice
1849
# is given in the value
1850
return get_choice(request, value, name, list(default.argtype),
1853
return _convert_arg(request, value, default.argtype, name)
1856
assert isinstance(fixed_args, (list, tuple))
1865
assert isinstance(args, unicode)
1867
positional, keyword, trailing = parse_quoted_separated(args)
1871
kwargs[str(kw)] = keyword[kw]
1872
except UnicodeEncodeError:
1873
kwargs_to_pass[kw] = keyword[kw]
1875
trailing_args.extend(trailing)
1880
if isfunction(function) or ismethod(function):
1881
argnames, varargs, varkw, defaultlist = getargspec(function)
1882
elif isclass(function):
1884
varkw, defaultlist) = getargspec(function.__init__.im_func)
1886
raise TypeError('function must be a function, method or class')
1889
if ismethod(function) or isclass(function):
1890
argnames = argnames[1:]
1892
fixed_argc = len(fixed_args)
1893
argnames = argnames[fixed_argc:]
1894
argc = len(argnames)
1898
# if the fixed parameters have defaults too...
1899
if argc < len(defaultlist):
1900
defaultlist = defaultlist[fixed_argc:]
1901
defstart = argc - len(defaultlist)
1904
# reverse to be able to pop() things off
1905
positional.reverse()
1906
allow_kwargs = False
1907
allow_trailing = False
1908
# convert all arguments to keyword arguments,
1909
# fill all arguments that weren't given with None
1910
for idx in range(argc):
1911
argname = argnames[idx]
1912
if argname == '_kwargs':
1915
if argname == '_trailing_args':
1916
allow_trailing = True
1919
kwargs[argname] = positional.pop()
1920
if not argname in kwargs:
1921
kwargs[argname] = None
1923
defaults[argname] = defaultlist[idx - defstart]
1926
if not allow_trailing:
1927
raise ValueError(_('Too many arguments'))
1928
trailing_args.extend(positional)
1931
if not allow_trailing:
1932
raise ValueError(_('Cannot have arguments without name following'
1933
' named arguments'))
1934
kwargs['_trailing_args'] = trailing_args
1936
# type-convert all keyword arguments to the type
1937
# that the default value indicates
1938
for argname in kwargs.keys()[:]:
1939
if argname in defaults:
1940
# the value of 'argname' from kwargs will be put into the
1941
# macro's 'argname' argument, so convert that giving the
1942
# name to the converter so the user is told which argument
1943
# went wrong (if it does)
1944
kwargs[argname] = _convert_arg(request, kwargs[argname],
1945
defaults[argname], argname)
1946
if kwargs[argname] is None:
1947
if isinstance(defaults[argname], required_arg):
1948
raise ValueError(_('Argument "%s" is required') % argname)
1949
if isinstance(defaults[argname], IEFArgument):
1950
kwargs[argname] = defaults[argname].get_default()
1952
if not argname in argnames:
1953
# move argname into _kwargs parameter
1954
kwargs_to_pass[argname] = kwargs[argname]
1958
kwargs['_kwargs'] = kwargs_to_pass
1959
if not allow_kwargs:
1960
raise ValueError(_(u'No argument named "%s"') % (
1961
kwargs_to_pass.keys()[0]))
1963
return function(*fixed_args, **kwargs)
1966
def parseAttributes(request, attrstring, endtoken=None, extension=None):
1968
Parse a list of attributes and return a dict plus a possible
1970
If extension is passed, it has to be a callable that returns
1971
a tuple (found_flag, msg). found_flag is whether it did find and process
1972
something, msg is '' when all was OK or any other string to return an error
1975
@param request: the request object
1976
@param attrstring: string containing the attributes to be parsed
1977
@param endtoken: token terminating parsing
1978
@param extension: extension function -
1979
gets called with the current token, the parser and the dict
1981
@return: a dict plus a possible error message
1983
import shlex, StringIO
1987
parser = shlex.shlex(StringIO.StringIO(attrstring))
1988
parser.commenters = ''
1994
key = parser.get_token()
1995
except ValueError, err:
2000
if endtoken and key == endtoken:
2003
# call extension function with the current token, the parser, and the dict
2005
found_flag, msg = extension(key, parser, attrs)
2006
#logging.debug("%r = extension(%r, parser, %r)" % (msg, key, attrs))
2011
#else (we found nothing, but also didn't have an error msg) we just continue below:
2014
eq = parser.get_token()
2015
except ValueError, err:
2019
msg = _('Expected "=" to follow "%(token)s"') % {'token': key}
2023
val = parser.get_token()
2024
except ValueError, err:
2028
msg = _('Expected a value for key "%(token)s"') % {'token': key}
2031
key = escape(key) # make sure nobody cheats
2033
# safely escape and quote value
2034
if val[0] in ["'", '"']:
2037
val = '"%s"' % escape(val, 1)
2039
attrs[key.lower()] = val
2041
return attrs, msg or ''
2044
class ParameterParser:
2045
""" MoinMoin macro parameter parser
2047
Parses a given parameter string, separates the individual parameters
2048
and detects their type.
2050
Possible parameter types are:
2052
Name | short | example
2053
----------------------------
2055
Float | f | 234.234 23.345E-23
2056
String | s | 'Stri\'ng'
2057
Boolean | b | 0 1 True false
2058
Name | | case_sensitive | converted to string
2060
So say you want to parse three things, name, age and if the
2061
person is male or not:
2063
The pattern will be: %(name)s%(age)i%(male)b
2065
As a result, the returned dict will put the first value into
2066
male, second into age etc. If some argument is missing, it will
2067
get None as its value. This also means that all the identifiers
2068
in the pattern will exist in the dict, they will just have the
2069
value None if they were not specified by the caller.
2071
So if we call it with the parameters as follows:
2073
this will result in the following dict:
2074
{"name": "John Smith", "age": 18, "male": None}
2076
Another way of calling would be:
2077
("John Smith", male=True)
2078
this will result in the following dict:
2079
{"name": "John Smith", "age": None, "male": True}
2082
def __init__(self, pattern):
2083
# parameter_re = "([^\"',]*(\"[^\"]*\"|'[^']*')?[^\"',]*)[,)]"
2084
name = "(?P<%s>[a-zA-Z_][a-zA-Z0-9_]*)"
2085
int_re = r"(?P<int>-?\d+)"
2086
bool_re = r"(?P<bool>(([10])|([Tt]rue)|([Ff]alse)))"
2087
float_re = r"(?P<float>-?\d+\.\d+([eE][+-]?\d+)?)"
2088
string_re = (r"(?P<string>('([^']|(\'))*?')|" +
2089
r'("([^"]|(\"))*?"))')
2090
name_re = name % "name"
2091
name_param_re = name % "name_param"
2093
param_re = r"\s*(\s*%s\s*=\s*)?(%s|%s|%s|%s|%s)\s*(,|$)" % (
2094
name_re, float_re, int_re, bool_re, string_re, name_param_re)
2095
self.param_re = re.compile(param_re, re.U)
2096
self._parse_pattern(pattern)
2098
def _parse_pattern(self, pattern):
2099
param_re = r"(%(?P<name>\(.*?\))?(?P<type>[ibfs]{1,3}))|\|"
2101
# TODO: Optionals aren't checked.
2104
self.param_list = []
2105
self.param_dict = {}
2107
for match in re.finditer(param_re, pattern):
2108
if match.group() == "|":
2109
self.optional.append(i)
2111
self.param_list.append(match.group('type'))
2112
if match.group('name'):
2114
self.param_dict[match.group('name')[1:-1]] = i
2116
raise ValueError("Named parameter expected")
2120
return "%s, %s, optional:%s" % (self.param_list, self.param_dict,
2123
def parse_parameters(self, params):
2124
# Default list/dict entries to None
2125
parameter_list = [None] * len(self.param_list)
2126
parameter_dict = dict([(key, None) for key in self.param_dict])
2127
check_list = [0] * len(self.param_list)
2134
while start < len(params):
2135
match = re.match(self.param_re, params[start:])
2137
raise ValueError("malformed parameters")
2138
start += match.end()
2139
if match.group("int"):
2140
pvalue = int(match.group("int"))
2142
elif match.group("bool"):
2143
pvalue = (match.group("bool") == "1") or (match.group("bool") == "True") or (match.group("bool") == "true")
2145
elif match.group("float"):
2146
pvalue = float(match.group("float"))
2148
elif match.group("string"):
2149
pvalue = match.group("string")[1:-1]
2151
elif match.group("name_param"):
2152
pvalue = match.group("name_param")
2155
raise ValueError("Parameter parser code does not fit param_re regex")
2157
name = match.group("name")
2159
if name not in self.param_dict:
2160
# TODO we should think on inheritance of parameters
2161
raise ValueError("unknown parameter name '%s'" % name)
2162
nr = self.param_dict[name]
2164
raise ValueError("parameter '%s' specified twice" % name)
2167
pvalue = self._check_type(pvalue, ptype, self.param_list[nr])
2168
parameter_dict[name] = pvalue
2169
parameter_list[nr] = pvalue
2172
raise ValueError("only named parameters allowed after first named parameter")
2175
if nr not in self.param_dict.values():
2176
fixed_count = nr + 1
2177
parameter_list[nr] = self._check_type(pvalue, ptype, self.param_list[nr])
2179
# Let's populate and map our dictionary to what's been found
2180
for name in self.param_dict:
2181
tmp = self.param_dict[name]
2182
parameter_dict[name] = parameter_list[tmp]
2186
for i in range(fixed_count):
2187
parameter_dict[i] = parameter_list[i]
2189
return fixed_count, parameter_dict
2191
def _check_type(self, pvalue, ptype, format):
2192
if ptype == 'n' and 's' in format: # n as s
2196
return pvalue # x -> x
2200
return float(pvalue) # i -> f
2202
return pvalue != 0 # i -> b
2205
if pvalue.lower() == 'false':
2206
return False # s-> b
2207
elif pvalue.lower() == 'true':
2210
raise ValueError('%r does not match format %r' % (pvalue, format))
2212
if 's' in format: # * -> s
2215
raise ValueError('%r does not match format %r' % (pvalue, format))
2218
#############################################################################
2220
#############################################################################
2221
def normalize_pagename(name, cfg):
2222
""" Normalize page name
2224
Prevent creating page names with invisible characters or funny
2225
whitespace that might confuse the users or abuse the wiki, or
2226
just does not make sense.
2228
Restrict even more group pages, so they can be used inside acl lines.
2230
@param name: page name, unicode
2232
@return: decoded and sanitized page name
2234
# Strip invalid characters
2235
name = config.page_invalid_chars_regex.sub(u'', name)
2237
# Split to pages and normalize each one
2238
pages = name.split(u'/')
2241
# Ignore empty or whitespace only pages
2242
if not page or page.isspace():
2245
# Cleanup group pages.
2246
# Strip non alpha numeric characters, keep white space
2247
if isGroupPage(page, cfg):
2248
page = u''.join([c for c in page
2249
if c.isalnum() or c.isspace()])
2251
# Normalize white space. Each name can contain multiple
2252
# words separated with only one space. Split handle all
2253
# 30 unicode spaces (isspace() == True)
2254
page = u' '.join(page.split())
2256
normalized.append(page)
2258
# Assemble components into full pagename
2259
name = u'/'.join(normalized)
2262
def taintfilename(basename):
2264
Make a filename that is supposed to be a plain name secure, i.e.
2265
remove any possible path components that compromise our system.
2267
@param basename: (possibly unsafe) filename
2269
@return: (safer) filename
2271
for x in (os.pardir, ':', '/', '\\', '<', '>'):
2272
basename = basename.replace(x, '_')
2277
def drawing2fname(drawing):
2278
config.drawing_extensions = ['.tdraw', '.adraw',
2280
'.png', '.jpg', '.jpeg', '.gif',
2282
fname, ext = os.path.splitext(drawing)
2283
# note: do not just check for empty extension or stuff like drawing:foo.bar
2284
# will fail, instead of being expanded to foo.bar.tdraw
2285
if ext not in config.drawing_extensions:
2286
# for backwards compatibility, twikidraw is the default:
2291
def mapURL(request, url):
2293
Map URLs according to 'cfg.url_mappings'.
2299
# check whether we have to map URLs
2300
if request.cfg.url_mappings:
2301
# check URL for the configured prefixes
2302
for prefix in request.cfg.url_mappings:
2303
if url.startswith(prefix):
2304
# substitute prefix with replacement value
2305
return request.cfg.url_mappings[prefix] + url[len(prefix):]
2307
# return unchanged url
2311
def getUnicodeIndexGroup(name):
2313
Return a group letter for `name`, which must be a unicode string.
2314
Currently supported: Hangul Syllables (U+AC00 - U+D7AF)
2316
@param name: a string
2318
@return: group letter or None
2321
if u'\uAC00' <= c <= u'\uD7AF': # Hangul Syllables
2322
return unichr(0xac00 + (int(ord(c) - 0xac00) / 588) * 588)
2324
return c.upper() # we put lower and upper case words into the same index group
2327
def isStrictWikiname(name, word_re=re.compile(ur"^(?:[%(u)s][%(l)s]+){2,}$" % {'u': config.chars_upper, 'l': config.chars_lower})):
2329
Check whether this is NOT an extended name.
2331
@param name: the wikiname in question
2333
@return: true if name matches the word_re
2335
return word_re.match(name)
2338
def is_URL(arg, schemas=config.url_schemas):
2339
""" Return True if arg is a URL (with a schema given in the schemas list).
2341
Note: there are not that many requirements for generic URLs, basically
2342
the only mandatory requirement is the ':' between schema and rest.
2343
Schema itself could be anything, also the rest (but we only support some
2344
schemas, as given in config.url_schemas, so it is a bit less ambiguous).
2348
for schema in schemas:
2349
if arg.startswith(schema + ':'):
2356
Is this a picture's url?
2358
@param url: the url in question
2360
@return: true if url points to a picture
2362
extpos = url.rfind(".") + 1
2363
return extpos > 1 and url[extpos:].lower() in config.browser_supported_images
2366
def link_tag(request, params, text=None, formatter=None, on=None, **kw):
2369
TODO: cleanup css_class
2371
@param request: the request object
2372
@param params: parameter string appended to the URL after the scriptname/
2373
@param text: text / inner part of the <a>...</a> link - does NOT get
2374
escaped, so you can give HTML here and it will be used verbatim
2375
@param formatter: the formatter object to use
2376
@param on: opening/closing tag only
2377
@keyword attrs: additional attrs (HTMLified string) (removed in 1.5.3)
2379
@return: formatted link tag
2381
if formatter is None:
2382
formatter = request.html_formatter
2383
if 'css_class' in kw:
2384
css_class = kw['css_class']
2385
del kw['css_class'] # one time is enough
2388
id = kw.get('id', None)
2389
name = kw.get('name', None)
2391
text = params # default
2393
url = "%s/%s" % (request.script_root, params)
2394
# formatter.url will escape the url part
2396
tag = formatter.url(on, url, css_class, **kw)
2398
tag = (formatter.url(1, url, css_class, **kw) +
2399
formatter.rawHTML(text) +
2401
else: # this shouldn't be used any more:
2402
if on is not None and not on:
2407
attrs += ' class="%s"' % css_class
2409
attrs += ' id="%s"' % id
2411
attrs += ' name="%s"' % name
2412
tag = '<a%s href="%s/%s">' % (attrs, request.script_root, params)
2414
tag = "%s%s</a>" % (tag, text)
2415
logging.warning("wikiutil.link_tag called without formatter and without request.html_formatter. tag=%r" % (tag, ))
2418
def containsConflictMarker(text):
2419
""" Returns true if there is a conflict marker in the text. """
2420
return "/!\\ '''Edit conflict" in text
2422
def pagediff(request, pagename1, rev1, pagename2, rev2, **kw):
2424
Calculate the "diff" between two page contents.
2426
@param pagename1: name of first page
2427
@param rev1: revision of first page
2428
@param pagename2: name of second page
2429
@param rev2: revision of second page
2430
@keyword ignorews: if 1: ignore pure-whitespace changes.
2432
@return: lines of diff output
2434
from MoinMoin.Page import Page
2435
from MoinMoin.util import diff_text
2436
lines1 = Page(request, pagename1, rev=rev1).getlines()
2437
lines2 = Page(request, pagename2, rev=rev2).getlines()
2439
lines = diff_text.diff(lines1, lines2, **kw)
2442
def anchor_name_from_text(text):
2444
Generate an anchor name from the given text.
2445
This function generates valid HTML IDs matching: [A-Za-z][A-Za-z0-9:_.-]*
2446
Note: this transformation has a special feature: when you feed it with a
2447
valid ID/name, it will return it without modification (identity
2450
quoted = urllib.quote_plus(text.encode('utf-7'), safe=':')
2451
res = quoted.replace('%', '.').replace('+', '_')
2452
if not res[:1].isalpha():
2456
def split_anchor(pagename):
2458
Split a pagename that (optionally) has an anchor into the real pagename
2459
and the anchor part. If there is no anchor, it returns an empty string
2462
Note: if pagename contains a # (as part of the pagename, not as anchor),
2463
you can use a trick to make it work nevertheless: just append a
2465
"C##" returns ("C#", "")
2466
"Problem #1#" returns ("Problem #1", "")
2468
TODO: We shouldn't deal with composite pagename#anchor strings, but keep
2470
Current approach: [[pagename#anchor|label|attr=val,&qarg=qval]]
2471
Future approach: [[pagename|label|attr=val,&qarg=qval,#anchor]]
2472
The future approach will avoid problems when there is a # in the
2473
pagename part (and no anchor). Also, we need to append #anchor
2474
at the END of the generated URL (AFTER the query string).
2476
parts = rsplit(pagename, '#', 1)
2482
########################################################################
2483
### Tickets - usually used in forms to make sure that form submissions
2484
### are in response to a form the same user got from moin for the same
2485
### action and same page.
2486
########################################################################
2488
def createTicket(request, tm=None, action=None, pagename=None):
2489
""" Create a ticket using a configured secret
2491
@param tm: unix timestamp (optional, uses current time if not given)
2492
@param action: action name (optional, uses current action if not given)
2493
Note: if you create a ticket for a form that calls another
2494
action than the current one, you MUST specify the
2495
action you call when posting the form.
2496
@param pagename: page name (optional, uses current page name if not given)
2497
Note: if you create a ticket for a form that posts to another
2498
page than the current one, you MUST specify the
2499
page name you use when posting the form.
2502
from MoinMoin.support.python_compatibility import hmac_new
2504
# for age-check of ticket
2505
tm = "%010x" % time.time()
2507
# make the ticket very specific:
2508
if pagename is None:
2510
pagename = request.page.page_name
2515
action = request.action
2518
# either a user is logged in or we have a anon session -
2519
# if session times out, ticket will get invalid
2520
sid = request.session.sid
2524
if request.user.valid:
2525
uid = request.user.id
2530
for value in [tm, pagename, action, sid, uid, ]:
2531
if isinstance(value, unicode):
2532
value = value.encode('utf-8')
2533
hmac_data.append(value)
2535
hmac = hmac_new(request.cfg.secrets['wikiutil/tickets'],
2537
return "%s.%s" % (tm, hmac.hexdigest())
2540
def checkTicket(request, ticket):
2541
"""Check validity of a previously created ticket"""
2543
timestamp_str = ticket.split('.')[0]
2544
timestamp = int(timestamp_str, 16)
2546
# invalid or empty ticket
2547
logging.debug("checkTicket: invalid or empty ticket %r" % ticket)
2550
if timestamp < now - 10 * 3600:
2551
# we don't accept tickets older than 10h
2552
logging.debug("checkTicket: too old ticket, timestamp %r" % timestamp)
2554
# Note: if the session timed out, that will also invalidate the ticket,
2555
# if the ticket was created within a session.
2556
ourticket = createTicket(request, timestamp_str)
2557
logging.debug("checkTicket: returning %r, got %r, expected %r" % (ticket == ourticket, ticket, ourticket))
2558
return safe_str_equal(ticket, ourticket)
2561
def renderText(request, Parser, text):
2562
"""executes raw wiki markup with all page elements"""
2564
out = StringIO.StringIO()
2565
request.redirect(out)
2566
wikiizer = Parser(text, request)
2567
wikiizer.format(request.formatter, inhibit_p=True)
2568
result = out.getvalue()
2573
def get_processing_instructions(body):
2574
""" Extract the processing instructions / acl / etc. at the beginning of a page's body.
2576
Hint: if you have a Page object p, you already have the result of this function in
2577
p.meta and (even better) parsed/processed stuff in p.pi.
2579
Returns a list of (pi, restofline) tuples and a string with the rest of the body.
2582
while body.startswith('#'):
2584
line, body = body.split('\n', 1) # extract first line
2589
# end parsing on empty (invalid) PI
2591
body = line + '\n' + body
2594
if line[1] == '#':# two hash marks are a comment
2596
if not comment.startswith(' '):
2597
# we don't require a blank after the ##, so we put one there
2598
comment = ' ' + comment
2599
line = '##%s' % comment
2601
verb, args = (line[1:] + ' ').split(' ', 1) # split at the first blank
2602
pi.append((verb.lower(), args.strip()))
2607
class Version(tuple):
2609
Version objects store versions like 1.2.3-4.5alpha6 in a structured
2610
way and support version comparisons and direct version component access.
2611
1: major version (digits only)
2612
2: minor version (digits only)
2613
3: (maintenance) release version (digits only)
2614
4.5alpha6: optional additional version specification (str)
2616
You can create a Version instance either by giving the components, like:
2617
Version(1,2,3,'4.5alpha6')
2618
or by giving the composite version string, like:
2619
Version(version="1.2.3-4.5alpha6").
2621
Version subclasses tuple, so comparisons to tuples should work.
2622
Also, we inherit all the comparison logic from tuple base class.
2624
VERSION_RE = re.compile(
2636
def parse_version(cls, version):
2637
match = cls.VERSION_RE.match(version)
2639
raise ValueError("Unexpected version string format: %r" % version)
2640
v = match.groupdict()
2641
return int(v['major']), int(v['minor']), int(v['release']), str(v['additional'] or '')
2643
def __new__(cls, major=0, minor=0, release=0, additional='', version=None):
2645
major, minor, release, additional = cls.parse_version(version)
2646
return tuple.__new__(cls, (major, minor, release, additional))
2648
# properties for easy access of version components
2649
major = property(lambda self: self[0])
2650
minor = property(lambda self: self[1])
2651
release = property(lambda self: self[2])
2652
additional = property(lambda self: self[3])
2655
version_str = "%d.%d.%d" % (self.major, self.minor, self.release)
2657
version_str += "-%s" % self.additional
added
removed
.pc/secure_taintfile_name.patch/MoinMoin/wikiutil.py
