1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
|
# Copyright (C) 2009-2012 by the Free Software Foundation, Inc.
#
# This file is part of GNU Mailman.
#
# GNU Mailman is free software: you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option)
# any later version.
#
# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.
#
# You should have received a copy of the GNU General Public License along with
# GNU Mailman. If not, see <http://www.gnu.org/licenses/>.
"""String utilities."""
from __future__ import absolute_import, unicode_literals
__metaclass__ = type
__all__ = [
'expand',
'oneline',
'uncanonstr',
'websafe',
'wrap',
]
import cgi
import logging
from email.errors import HeaderParseError
from email.header import decode_header, make_header
from string import Template, whitespace
from textwrap import TextWrapper, dedent
from zope.component import getUtility
from mailman.interfaces.languages import ILanguageManager
EMPTYSTRING = ''
NL = '\n'
log = logging.getLogger('mailman.error')
def expand(template, substitutions, template_class=Template):
"""Expand string template with substitutions.
:param template: A PEP 292 $-string template.
:type template: string
:param substitutions: The substitutions dictionary.
:type substitutions: dict
:param template_class: The template class to use.
:type template_class: class
:return: The substituted string.
:rtype: string
"""
return template_class(template).safe_substitute(substitutions)
def oneline(s, cset='us-ascii', in_unicode=False):
"""Decode a header string in one line and convert into specified charset.
:param s: The header string
:type s: string
:param cset: The character set (encoding) to use.
:type cset: string
:param in_unicode: Flag specifying whether to return the converted string
as a unicode (True) or an 8-bit string (False, the default).
:type in_unicode: bool
:return: The decoded header string. If an error occurs while converting
the input string, return the string undecoded, as an 8-bit string.
:rtype: string
"""
try:
h = make_header(decode_header(s))
ustr = h.__unicode__()
line = EMPTYSTRING.join(ustr.splitlines())
if in_unicode:
return line
else:
return line.encode(cset, 'replace')
except (LookupError, UnicodeError, ValueError, HeaderParseError):
# possibly charset problem. return with undecoded string in one line.
return EMPTYSTRING.join(s.splitlines())
def websafe(s):
return cgi.escape(s, quote=True)
# The opposite of canonstr() -- sorta. I.e. it attempts to encode s in the
# charset of the given language, which is the character set that the page will
# be rendered in, and failing that, replaces non-ASCII characters with their
# html references. It always returns a byte string.
def uncanonstr(s, lang=None):
if s is None:
s = u''
if lang is None:
charset = 'us-ascii'
else:
charset = getUtility(ILanguageManager)[lang].charset
# See if the string contains characters only in the desired character
# set. If so, return it unchanged, except for coercing it to a byte
# string.
try:
if isinstance(s, unicode):
return s.encode(charset)
else:
unicode(s, charset)
return s
except UnicodeError:
# Nope, it contains funny characters, so html-ref it
a = []
for c in s:
o = ord(c)
if o > 127:
a.append('&#%3d;' % o)
else:
a.append(c)
# Join characters together and coerce to byte string
return str(EMPTYSTRING.join(a))
def wrap(text, column=70, honor_leading_ws=True):
"""Wrap and fill the text to the specified column.
The input text is wrapped and filled as done by the standard library
textwrap module. The differences here being that this function is capable
of filling multiple paragraphs (as defined by text separated by blank
lines). Also, when `honor_leading_ws` is True (the default), paragraphs
that being with whitespace are not wrapped. This is the algorithm that
the Python FAQ wizard used.
"""
# First, split the original text into paragraph, keeping all blank lines
# between them.
paragraphs = []
paragraph = []
last_indented = False
for line in text.splitlines(True):
is_indented = (len(line) > 0 and line[0] in whitespace)
if line == NL:
if len(paragraph) > 0:
paragraphs.append(EMPTYSTRING.join(paragraph))
paragraphs.append(line)
last_indented = False
paragraph = []
elif last_indented != is_indented:
# The indentation level changed. We treat this as a paragraph
# break but no blank line will be issued between paragraphs.
if len(paragraph) > 0:
paragraphs.append(EMPTYSTRING.join(paragraph))
# The next paragraph starts with this line.
paragraph = [line]
last_indented = is_indented
else:
# This line does not constitute a paragraph break.
paragraph.append(line)
# We've consumed all the lines in the original text. Transfer the last
# paragraph we were collecting to the full set of paragraphs.
paragraphs.append(EMPTYSTRING.join(paragraph))
# Now iterate through all paragraphs, wrapping as necessary.
wrapped_paragraphs = []
# The dedented wrapper.
wrapper = TextWrapper(width=column,
fix_sentence_endings=True)
# The indented wrapper. For this one, we'll clobber initial_indent and
# subsequent_indent as needed per indented chunk of text.
iwrapper = TextWrapper(width=column,
fix_sentence_endings=True,
)
add_paragraph_break = False
for paragraph in paragraphs:
if add_paragraph_break:
wrapped_paragraphs.append(NL)
add_paragraph_break = False
paragraph_text = EMPTYSTRING.join(paragraph)
# Just copy the blank lines to the final set of paragraphs.
if len(paragraph) == 0 or paragraph == NL:
wrapped_paragraphs.append(NL)
# Choose the wrapper based on whether the paragraph is indented or
# not. Also, do not wrap indented paragraphs if honor_leading_ws is
# set.
elif paragraph[0] in whitespace:
if honor_leading_ws:
# Leave the indented paragraph verbatim.
wrapped_paragraphs.append(paragraph_text)
else:
# The paragraph should be wrapped, but it must first be
# dedented. The leading whitespace on the first line of the
# original text will be used as the indentation for all lines
# in the wrapped text.
for i, ch in enumerate(paragraph_text):
if ch not in whitespace:
break
leading_ws = paragraph[:i]
iwrapper.initial_indent=leading_ws
iwrapper.subsequent_indent=leading_ws
paragraph_text = dedent(paragraph_text)
wrapped_paragraphs.append(iwrapper.fill(paragraph_text))
add_paragraph_break = True
else:
# Fill this paragraph. fill() consumes the trailing newline.
wrapped_paragraphs.append(wrapper.fill(paragraph_text))
add_paragraph_break = True
return EMPTYSTRING.join(wrapped_paragraphs)
|