← Back to branch summary

~simplestreams-dev/simplestreams/artful

« back to all changes in this revision

Viewing changes to simplestreams/util.py

  • Committer: Package Import Robot
  • Author(s): Scott Moser
  • Date: 2014-08-22 10:06:19 UTC
  • mfrom: (1.1.14)
  • Revision ID: package-import@ubuntu.com-20140822100619-thgyplp9ch67sk8m
Tags: 0.1.0~bzr354-0ubuntu1
https://launchpad.net/bugs/1340983
* New upstream snapshot.
  - support progress output in the glance mirror (LP: #1340983)
  - upstream uses pyflakes instead of pylint

Show diffs side-by-side

added added

removed removed

Lines of Context:
simplestreams/util.py
30
30
try:
31
31
    ALGORITHMS = list(getattr(hashlib, 'algorithms'))
32
32
except AttributeError:
33
 
    ALGORITHMS = list(hashlib.algorithms_available)  # pylint: disable=E1101
 
33
    ALGORITHMS = list(hashlib.algorithms_available)
34
34
 
35
35
ALIASNAME = "_aliases"
36
36
 
249
249
    return(add, remove)
250
250
 
251
251
 
252
 
def policy_read_signed(content, path, keyring=None):  # pylint: disable=W0613
 
252
def policy_read_signed(content, path, keyring=None):
253
253
    # convenience wrapper around 'read_signed' for use MirrorReader policy
254
254
    return read_signed(content=content, keyring=keyring)
255
255
 
264
264
            cmd.append("--keyring=%s" % keyring)
265
265
        cmd.append("-")
266
266
        try:
267
 
            _outerr = subp(cmd, data=content)
 
267
            subp(cmd, data=content)
268
268
        except subprocess.CalledProcessError as e:
269
269
            LOG.debug("failed: %s\n out=%s\n err=%s" %
270
270
                      (' '.join(cmd), e.output[0], e.output[1]))
391
391
    # walk a products tree, copying up item keys as far as they'll go
392
392
    # only move items to a sibling of the 'top'.
393
393
 
394
 
    top_values = ('versions', 'products', None)
395
394
    if top not in ('versions', 'products'):
396
395
        raise ValueError("'top' must be one of: %s" %
397
396
                         ','.join(PRODUCTS_TREE_HIERARCHY))
437
436
    return
438
437
 
439
438
 
440
 
def get_local_copy(contentsource, read_size=READ_SIZE):
 
439
def get_local_copy(contentsource, read_size=READ_SIZE, progress_callback=None):
441
440
    (tfd, tpath) = tempfile.mkstemp()
442
441
    tfile = os.fdopen(tfd, "wb")
443
442
    try:
444
443
        LOG.debug("getting local copy of %s", contentsource.url)
445
444
        while True:
446
445
            buf = contentsource.read(read_size)
 
446
            if progress_callback:
 
447
                progress_callback(read_size)
447
448
            tfile.write(buf)
448
449
            if len(buf) != read_size:
449
450
                break
471
472
    if isinstance(err, bytes):
472
473
        err = err.decode('utf-8')
473
474
 
474
 
    rc = sp.returncode  # pylint: disable=E1101
 
475
    rc = sp.returncode
475
476
    if rc != 0:
476
477
        raise subprocess.CalledProcessError(rc, args, output=(out, err))
477
478
 
565
566
 
566
567
    return (mirror, path)
567
568
 
 
569
 
 
570
class ProgressAggregator(object):
 
571
    def __init__(self, remaining_items=None):
 
572
        self.last_emitted = 0
 
573
        self.current_written = 0
 
574
        self.current_file = None
 
575
        self.remaining_items = remaining_items
 
576
        if self.remaining_items:
 
577
            self.total_image_count = len(self.remaining_items)
 
578
            self.total_size = sum(self.remaining_items.values())
 
579
        else:
 
580
            self.total_image_count = 0
 
581
            self.total_size = 0
 
582
        self.total_written = 0
 
583
 
 
584
    def progress_callback(self, progress):
 
585
        if self.current_file != progress['name']:
 
586
            if self.remaining_items and self.current_file is not None:
 
587
                del self.remaining_items[self.current_file]
 
588
            self.current_file = progress['name']
 
589
            self.last_emitted = 0
 
590
            self.current_written = 0
 
591
 
 
592
        size = float(progress['size'])
 
593
        written = float(progress['written'])
 
594
        self.current_written += written
 
595
        self.total_written += written
 
596
        interval = self.current_written - self.last_emitted
 
597
        if interval > size / 100:
 
598
            self.last_emitted = self.current_written
 
599
            progress['written'] = self.current_written
 
600
            self.emit(progress)
 
601
 
 
602
    def emit(self, progress):
 
603
        raise NotImplementedError()
 
604
 
568
605
# vi: ts=4 expandtab