2
# -*- coding: utf-8 -*-
3
"""Process new requests to download per-package data"""
4
# Copyright (C) 2012 Canonical Ltd
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of version 3 of the GNU General Public License as
8
# published by the Free Software Foundation.
10
# This program is distributed in the hope that it will be useful,
11
# but WITHOUT ANY WARRANTY; without even the implied warranty of
12
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
# GNU General Public License for more details.
15
# You should have received a copy of the GNU General Public License along
16
# with this program; if not, write to the Free Software Foundation, Inc.,
17
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
27
from datetime import datetime
29
# avoid hanging forever (LP: #1243090)
31
socket.setdefaulttimeout(60)
34
DATADIR = "/usr/share/package-data-downloads/"
35
STAMPDIR = "/var/lib/update-notifier/package-data-downloads/"
36
NOTIFIER_SOURCE_FILE = \
37
"/usr/share/update-notifier/package-data-downloads-failed"
38
NOTIFIER_FILE = "/var/lib/update-notifier/user.d/data-downloads-failed"
39
NOTIFIER_PERMANENT_SOURCE_FILE = NOTIFIER_SOURCE_FILE + '-permanently'
40
NOTIFIER_PERMANENT_FILE = NOTIFIER_FILE + '-permanently'
43
permanent_failures = []
46
def create_or_update_stampfile(file):
47
"""Create or update the indicated stampfile, and remove failure flags"""
54
traceback.print_exc(file=sys.stderr)
58
for ext in ('.failed', '.permanent-failure'):
59
if os.path.exists(file + ext):
63
def mark_hook_failed(hook_name, permanent=False):
64
"""Create a stampfile recording that a hook failed
66
We create separate stampfiles for failed hooks so we can
67
keep track of how long the hook has been failing and if the failure
68
should be considered permanent."""
71
filename = hook_name + '.permanent-failure'
73
filename = hook_name + '.failed'
75
failure_file = os.path.join(STAMPDIR, filename)
77
with open(failure_file, 'w'):
82
traceback.print_exc(file=sys.stderr)
84
for ext in ('', '.failed', '.permanent-failure'):
85
stampfile = hook_name + ext
86
if filename != stampfile \
87
and os.path.exists(os.path.join(STAMPDIR, stampfile)):
88
os.unlink(os.path.join(STAMPDIR, stampfile))
91
def hook_is_permanently_failed(hook_name):
92
"""Check if this hook has been marked as permanently failing.
94
If so, don't raise any more errors about it."""
96
failure_file = os.path.join(STAMPDIR, hook_name + '.permanent-failure')
97
return os.path.exists(failure_file)
100
def hook_aged_out(hook_name):
101
"""Check if this hook has been failing consistently for >= 3 days"""
103
failure_file = os.path.join(STAMPDIR, hook_name + '.failed')
105
hook_date = datetime.fromtimestamp(os.stat(failure_file).st_ctime)
106
cur_time = datetime.now()
107
d = cur_time - hook_date
113
traceback.print_exc(file=sys.stderr)
117
def record_failure(hook):
118
"""Record that the named hook has failed"""
119
if hook_aged_out(hook):
120
permanent_failures.append(hook)
122
failures.append(hook)
125
def existing_permanent_failures():
126
"""Return the list of all previously recorded permanent failures"""
128
files = glob.glob(os.path.join(STAMPDIR, "*.permanent-failure"))
129
return [os.path.splitext(os.path.basename(path))[0] for path in files]
132
def trigger_update_notifier(failures, permanent=False):
133
"""Tell update-notifier that there were failed packages"""
137
with open(NOTIFIER_PERMANENT_SOURCE_FILE, 'r',
138
encoding='utf-8') as f:
140
output_file = open(NOTIFIER_PERMANENT_FILE, 'w', encoding='utf-8')
142
with open(NOTIFIER_SOURCE_FILE, 'r', encoding='utf-8') as f:
144
output_file = open(NOTIFIER_FILE, 'w', encoding='utf-8')
146
# Things failed and we can't even notify about it. Break the
147
# trigger so that there's some error propagation, even if not
148
# the most pleasant sort.
149
traceback.print_exc(file=sys.stderr)
152
packages = [os.path.basename(failure) for failure in failures]
154
string.Template(input).substitute(
155
{'packages': ", ".join(packages)}))
159
def get_hook_file_names():
161
for relfile in os.listdir(DATADIR):
162
# ignore files ending in .dpkg-*
163
if (os.path.splitext(relfile)[1]
164
and os.path.splitext(relfile)[1].startswith(".dpkg")):
170
# we use apt-helper here as this gives us the exact same proxy behavior
171
# as apt-get itself (environment/apt-config proxy settings/autodiscover)
172
def download_file(uri, sha256_hashsum):
173
"""Download a URI and checks the given hashsum using apt-helper
175
Returns: path to the downloaded file or None
177
download_dir = os.path.join(STAMPDIR, "partial")
178
dest_file = os.path.join(download_dir, os.path.basename(uri))
179
ret = subprocess.call(
180
["/usr/lib/apt/apt-helper",
181
"download-file", uri, dest_file, "SHA256:" + sha256_hashsum])
183
if os.path.exists(dest_file):
189
def print_maybe(*args, **kwargs):
190
"""Version of print() that ignores failure"""
192
print(*args, **kwargs)
197
def process_download_requests():
198
"""Process requests to download package data files
200
Iterate over /usr/share/package-data-downloads and download any
201
package data specified in the contained file, then hand off to
202
the indicated handler for further processing.
204
Successful downloads are recorded in
205
/var/lib/update-notifier/package-data-downloads to avoid unnecessary
208
Failed downloads are reported to the user via the
209
update-notifier interface."""
211
# Iterate through all the available hooks. If we get a failure
212
# processing any of them (download failure, checksum failure, or
213
# failure to run the hook script), record it but continue processing
214
# the rest of the hooks since some of them may succeed.
215
for relfile in get_hook_file_names():
217
stampfile = os.path.join(STAMPDIR, relfile)
218
file = os.path.join(DATADIR, relfile)
220
if not os.path.exists(NOTIFIER_FILE) and \
221
not os.path.exists(NOTIFIER_PERMANENT_FILE):
222
hook_date = os.stat(file).st_mtime
223
stamp_date = os.stat(stampfile).st_mtime
224
if hook_date < stamp_date:
226
elif os.path.exists(stampfile):
229
except Exception as e:
230
if not isinstance(e, OSError):
231
traceback.print_exc(file=sys.stderr)
233
hook = debian.deb822.Deb822()
236
for para in hook.iter_paragraphs(open(file)):
239
record_failure(relfile)
241
command = [para['Script']]
243
if 'Should-Download' in para:
244
db = debconf.DebconfCommunicator('update-notifier')
246
should = db.get(para['Should-Download'])
247
if should == "false":
248
# Do nothing with this file.
250
except (DebconfError, KeyError):
255
print_maybe("%s: processing..." % (relfile))
257
# Download each file and verify the sum
260
for i in range(len(files)):
261
print_maybe("%s: downloading %s" % (relfile, files[i]))
262
dest_file = download_file(files[i], sums[i])
264
command.append(dest_file)
265
downloaded.add(dest_file)
267
record_failure(relfile)
269
if relfile in failures + permanent_failures:
273
result = subprocess.call(command)
275
# There's no sense redownloading if the script fails
276
permanent_failures.append(relfile)
278
create_or_update_stampfile(stampfile)
284
traceback.print_exc(file=sys.stderr)
286
record_failure(relfile)
287
# The 'script' is always the last stanza
290
# Not in a 'script' stanza, so we should have some urls
292
files.append(para['Url'])
293
sums.append(para['Sha256'])
294
except Exception as e:
295
print_maybe("%s: Error processing!" % (relfile))
296
if not isinstance(e, KeyError):
297
traceback.print_exc(file=sys.stderr)
298
record_failure(relfile)
301
previous_failures = existing_permanent_failures()
303
# We only report about "permanent" failures when there are new ones,
304
# but we want the whole list of permanently-failing hooks so when
305
# we clobber the update-notifier file we don't lose information the
306
# user may not have seen yet
307
if permanent_failures:
309
for failure in permanent_failures:
310
if failure not in previous_failures:
311
mark_hook_failed(failure, permanent=True)
312
previous_failures.append(failure)
315
trigger_update_notifier(previous_failures, permanent=True)
316
# 2016-09-19 14:36 reset the list of permanent_failures as it caused
317
# tests not to be idempotent
318
permanent_failures.clear()
319
if not previous_failures and os.path.exists(NOTIFIER_PERMANENT_FILE):
320
os.unlink(NOTIFIER_PERMANENT_FILE)
322
# Filter out new failure reports for permanently-failed packages
323
our_failures = [x for x in failures if x not in previous_failures]
324
# 2016-09-19 14:36 reset the list of permanent_failures as it caused
325
# tests not to be idempotent
329
for failure in our_failures:
330
mark_hook_failed(failure)
331
trigger_update_notifier(our_failures)
332
elif os.path.exists(NOTIFIER_FILE):
333
os.unlink(NOTIFIER_FILE)
336
if __name__ == "__main__":
337
process_download_requests()
added
removed
data/package-data-downloader
