1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
|
import cgi
from gzip import GzipFile
from htmlentitydefs import name2codepoint
import os
import os.path
from pkg_resources import resource_exists, resource_string
import re
from StringIO import StringIO
import time
from urllib import urlencode
import urllib2
import zlib
from html5lib import HTMLParser, treebuilders
from BeautifulSoup import BeautifulSoup
# json only in Python >=2.6
try:
import simplejson as json
except ImportError:
import json
# xml.etree only in Python >= 2.5
try:
from xml.etree import cElementTree as ElementTree
except ImportError:
import cElementTree as ElementTree
import ibid
def ago(delta, units=None):
parts = []
for unit, value in (
('year', delta.days/365), ('month', delta.days/30 % 12),
('day', delta.days % 30), ('hour', delta.seconds/3600),
('minute', delta.seconds/60 % 60), ('second', delta.seconds % 60),
('millisecond', delta.microseconds/1000)):
if value > 0 and (unit != 'millisecond' or len(parts) == 0):
parts.append('%s %s%s' % (value, unit, value != 1 and 's' or ''))
if units and len(parts) >= units:
break
formatted = ' and '.join(parts)
return formatted.replace(' and ', ', ', len(parts)-2)
def decode_htmlentities(text):
replace = lambda match: unichr(int(match.group(1)))
text = re.sub("&#(\d+);", replace, text)
replace = lambda match: match.group(1) in name2codepoint and unichr(name2codepoint[match.group(1)]) or match.group(0)
text = re.sub("&(\w+);", replace, text)
return text
def cacheable_download(url, cachefile):
"""Download url to cachefile if it's modified since cachefile.
Specify cachefile in the form pluginname/cachefile.
Returns complete path to downloaded file."""
# We do allow absolute paths, for people who know what they are doing,
# but the common use case should be pluginname/cachefile.
if cachefile[0] not in (os.sep, os.altsep):
cachedir = ibid.config.plugins['cachedir']
if not cachedir:
cachedir = os.path.join(ibid.options['base'], 'cache')
elif cachedir[0] == "~":
cachedir = os.path.expanduser(cachedir)
cachedir = os.path.abspath(cachedir)
plugindir = os.path.join(cachedir, os.path.dirname(cachefile))
if not os.path.isdir(plugindir):
os.makedirs(plugindir)
cachefile = os.path.join(cachedir, cachefile)
exists = os.path.isfile(cachefile)
req = urllib2.Request(url)
if exists:
modified = os.path.getmtime(cachefile)
modified = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(modified))
req.add_header("If-Modified-Since", modified)
try:
connection = urllib2.urlopen(req)
except urllib2.HTTPError, e:
if e.code == 304 and exists:
return cachefile
else:
raise
# Download into a temporary file, in case something goes wrong
downloadfile = os.path.join(plugindir, ".download." + os.path.basename(cachefile))
outfile = file(downloadfile, "wb")
data = connection.read()
compression = connection.headers.get('content-encoding')
if compression:
if compression.lower() == "deflate":
try:
data = zlib.decompress(data)
except zlib.error:
data = zlib.decompress(data, -zlib.MAX_WBITS)
elif compression.lower() == "gzip":
compressedstream = StringIO(data)
gzipper = GzipFile(fileobj=compressedstream)
data = gzipper.read()
outfile.write(data)
outfile.close()
try:
os.rename(downloadfile, cachefile)
except OSError:
# Are we on a system that doesn't support atomic renames?
os.remove(cachefile)
os.rename(downloadfile, cachefile)
return cachefile
def file_in_path(program):
path = os.environ.get("PATH", os.defpath).split(os.pathsep)
path = [os.path.join(dir, program) for dir in path]
path = [True for file in path if os.path.isfile(file)]
return bool(path)
def unicode_output(output, errors="strict"):
try:
encoding = os.getenv("LANG").split(".")[1]
except:
encoding = "ascii"
return unicode(output, encoding, errors)
def ibid_version():
return resource_exists(__name__, '.version') and resource_string(__name__, '.version').strip() or None
def get_html_parse_tree(url, data=None, headers={}, treetype='beautifulsoup'):
"Request a URL, parse with html5lib, and return a parse tree from it"
req = urllib2.Request(url, data, headers)
f = urllib2.urlopen(req)
data = f.read()
f.close()
encoding = None
contentType = f.headers.get('content-type')
if contentType:
(mediaType, params) = cgi.parse_header(contentType)
encoding = params.get('charset')
compression = f.headers.get('content-encoding')
if compression:
if compression.lower() == "deflate":
try:
data = zlib.decompress(data)
except zlib.error:
data = zlib.decompress(data, -zlib.MAX_WBITS)
elif compression.lower() == "gzip":
compressedstream = StringIO(data)
gzipper = GzipFile(fileobj=compressedstream)
data = gzipper.read()
if treetype == "beautifulsoup":
return BeautifulSoup(data, convertEntities=BeautifulSoup.HTML_ENTITIES)
elif treetype == "etree":
treebuilder = treebuilders.getTreeBuilder("etree", ElementTree)
else:
treebuilder = treebuilders.getTreeBuilder(treetype)
parser = HTMLParser(tree=treebuilder)
return parser.parse(data, encoding = encoding)
class JSONException(Exception):
pass
def json_webservice(url, params={}, headers={}):
"Request data from a JSON webservice, and deserialise"
for key in params:
if isinstance(params[key], unicode):
params[key] = params[key].encode('utf-8')
if params:
url += '?' + urlencode(params)
req = urllib2.Request(url, headers=headers)
f = urllib2.urlopen(req)
data = f.read()
f.close()
try:
return json.loads(data)
except ValueError, e:
raise JSONException(e)
# vi: set et sta sw=4 ts=4:
|