« back to all changes in this revision
Viewing changes to lava_dashboard_tool/commands.py
- browse files at revision 182
- compare with another revision
- download diff
- download tarball
- view history from revision 182
- Committer: Antonio Terceiro
- Date: 2013-04-30 00:56:51 UTC
- mfrom: (181.1.2 absorb-other-tools)
- Revision ID: antonio.terceiro@linaro.org-20130430005651-by4pgg1fynrkhjsn
Absorb lava-dashboard-tool and lava-scheduler-tool
- files added:
- lava_dashboard_tool
- lava_dashboard_tool/tests
- lava_scheduler_tool
- files modified:
- lava/tool/__init__.py
added
removed
lava_dashboard_tool/commands.py
1
# Copyright (C) 2010,2011 Linaro Limited
2
#
3
# Author: Zygmunt Krynicki <zygmunt.krynicki@linaro.org>
4
#
5
# This file is part of lava-dashboard-tool.
6
#
7
# lava-dashboard-tool is free software: you can redistribute it and/or modify
8
# it under the terms of the GNU Lesser General Public License version 3
9
# as published by the Free Software Foundation
10
#
11
# lava-dashboard-tool is distributed in the hope that it will be useful,
12
# but WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14
# GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU Lesser General Public License
17
# along with lava-dashboard-tool. If not, see <http://www.gnu.org/licenses/>.
18
19
"""
20
Module with command-line tool commands that interact with the dashboard
21
server. All commands listed here should have counterparts in the
22
launch_control.dashboard_app.xml_rpc package.
23
"""
24
25
import argparse
26
import contextlib
27
import errno
28
import os
29
import re
30
import socket
31
import sys
32
import urllib
33
import urlparse
34
import xmlrpclib
35
36
import simplejson
37
from json_schema_validator.extensions import datetime_extension
38
39
from lava_tool.authtoken import AuthenticatingServerProxy, KeyringAuthBackend
40
from lava.tool.commands import ExperimentalCommandMixIn
41
from lava.tool.command import Command, CommandGroup
42
43
44
class dashboard(CommandGroup):
45
"""
46
Commands for interacting with LAVA Dashboard
47
"""
48
49
namespace = "lava.dashboard.commands"
50
51
52
class InsufficientServerVersion(Exception):
53
"""
54
Exception raised when server version that a command interacts with is too
55
old to support required features.
56
"""
57
def __init__(self, server_version, required_version):
58
self.server_version = server_version
59
self.required_version = required_version
60
61
62
class DataSetRenderer(object):
63
"""
64
Support class for rendering a table out of list of dictionaries.
65
66
It supports several features that make printing tabular data easier.
67
* Automatic layout
68
* Custom column headers
69
* Custom cell formatting
70
* Custom table captions
71
* Custom column ordering
72
* Custom Column separators
73
* Custom dataset notification
74
75
The primary method is render() which does all of the above. You
76
need to pass a dataset argument which is a list of dictionaries.
77
Each dictionary must have the same keys. In particular the first row
78
is used to determine columns.
79
"""
80
def __init__(self, column_map=None, row_formatter=None, empty=None,
81
order=None, caption=None, separator=" ", header_separator=None):
82
if column_map is None:
83
column_map = {}
84
if row_formatter is None:
85
row_formatter = {}
86
if empty is None:
87
empty = "There is no data to display"
88
self.column_map = column_map
89
self.row_formatter = row_formatter
90
self.empty = empty
91
self.order = order
92
self.separator = separator
93
self.caption = caption
94
self.header_separator = header_separator
95
96
def _analyze_dataset(self, dataset):
97
"""
98
Determine the columns that will be displayed and the maximum
99
length of each of those columns.
100
101
Returns a tuple (dataset, columms, maxlen) where columns is a
102
list of column names and maxlen is a dictionary mapping from
103
column name to maximum length of any value in the row or the
104
column header and the dataset is a copy of the dataset altered
105
as necessary.
106
107
Some examples:
108
109
First the dataset, an array of dictionaries
110
>>> dataset = [
111
... {'a': 'shorter', 'bee': ''},
112
... {'a': 'little longer', 'bee': 'b'}]
113
114
Note that column 'bee' is actually three characters long as the
115
column name made it wider.
116
>>> dataset_out, columns, maxlen = DataSetRenderer(
117
... )._analyze_dataset(dataset)
118
119
Unless you format rows with a custom function the data is not altered.
120
>>> dataset_out is dataset
121
True
122
123
Columns come out in sorted alphabetic order
124
>>> columns
125
['a', 'bee']
126
127
Maximum length determines the width of each column. Note that
128
the header affects the column width.
129
>>> maxlen
130
{'a': 13, 'bee': 3}
131
132
You can constrain or reorder columns. In that case columns you
133
decided to ignore are simply left out of the output.
134
>>> dataset_out, columns, maxlen = DataSetRenderer(
135
... order=['bee'])._analyze_dataset(dataset)
136
>>> columns
137
['bee']
138
>>> maxlen
139
{'bee': 3}
140
141
You can format values anyway you like:
142
>>> dataset_out, columns, maxlen = DataSetRenderer(row_formatter={
143
... 'bee': lambda value: "%10s" % value}
144
... )._analyze_dataset(dataset)
145
146
Dataset is altered to take account of the row formatting
147
function. The original dataset argument is copied.
148
>>> dataset_out
149
[{'a': 'shorter', 'bee': ' '}, {'a': 'little longer', 'bee': ' b'}]
150
>>> dataset_out is not dataset
151
True
152
153
Columns stay the same though:
154
>>> columns
155
['a', 'bee']
156
157
Note how formatting altered the width of the column 'bee'
158
>>> maxlen
159
{'a': 13, 'bee': 10}
160
161
You can also format columns (with nice aliases).Note how
162
column 'bee' maximum width is now dominated by the long column
163
name:
164
>>> dataset_out, columns, maxlen = DataSetRenderer(column_map={
165
... 'bee': "Column B"})._analyze_dataset(dataset)
166
>>> maxlen
167
{'a': 13, 'bee': 8}
168
"""
169
if self.order:
170
columns = self.order
171
else:
172
columns = sorted(dataset[0].keys())
173
if self.row_formatter:
174
dataset_out = [dict(row) for row in dataset]
175
else:
176
dataset_out = dataset
177
for row in dataset_out:
178
for column in row:
179
if column in self.row_formatter:
180
row[column] = self.row_formatter[column](row[column])
181
maxlen = dict(
182
[(column, max(
183
len(self.column_map.get(column, column)),
184
max([
185
len(str(row[column])) for row in dataset_out])))
186
for column in columns])
187
return dataset_out, columns, maxlen
188
189
def _render_header(self, dataset, columns, maxlen):
190
"""
191
Render a header, possibly with a caption string
192
193
Caption is controlled by the constructor.
194
>>> dataset = [
195
... {'a': 'shorter', 'bee': ''},
196
... {'a': 'little longer', 'bee': 'b'}]
197
>>> columns = ['a', 'bee']
198
>>> maxlen = {'a': 13, 'bee': 3}
199
200
By default there is no caption, just column names:
201
>>> DataSetRenderer()._render_header(
202
... dataset, columns, maxlen)
203
a bee
204
205
If you enable the header separator then column names will be visually
206
separated from the first row of data.
207
>>> DataSetRenderer(header_separator=True)._render_header(
208
... dataset, columns, maxlen)
209
a bee
210
-----------------
211
212
If you provide a caption it gets rendered as a centered
213
underlined text before the data:
214
>>> DataSetRenderer(caption="Dataset")._render_header(
215
... dataset, columns, maxlen)
216
Dataset
217
=================
218
a bee
219
220
You can use both caption and header separator
221
>>> DataSetRenderer(caption="Dataset", header_separator=True)._render_header(
222
... dataset, columns, maxlen)
223
Dataset
224
=================
225
a bee
226
-----------------
227
228
Observe how the total length of the output horizontal line
229
depends on the separator! Also note the columns labels are
230
aligned to the center of the column
231
>>> DataSetRenderer(caption="Dataset", separator=" | ")._render_header(
232
... dataset, columns, maxlen)
233
Dataset
234
===================
235
a | bee
236
"""
237
total_len = sum(maxlen.itervalues())
238
if len(columns):
239
total_len += len(self.separator) * (len(columns) - 1)
240
# Print the caption
241
if self.caption:
242
print "{0:^{1}}".format(self.caption, total_len)
243
print "=" * total_len
244
# Now print the coulum names
245
print self.separator.join([
246
"{0:^{1}}".format(self.column_map.get(column, column),
247
maxlen[column]) for column in columns])
248
# Finally print the header separator
249
if self.header_separator:
250
print "-" * total_len
251
252
def _render_rows(self, dataset, columns, maxlen):
253
"""
254
Render rows of the dataset.
255
256
Each row is printed on one line using the maxlen argument to
257
determine correct column size. Text is aligned left.
258
259
First the dataset, columns and maxlen as produced by
260
_analyze_dataset()
261
>>> dataset = [
262
... {'a': 'shorter', 'bee': ''},
263
... {'a': 'little longer', 'bee': 'b'}]
264
>>> columns = ['a', 'bee']
265
>>> maxlen = {'a': 13, 'bee': 3}
266
267
Now a plain table. Note! To really understand this test
268
you should check out the length of the strings below. There
269
are two more spaces after 'b' in the second row
270
>>> DataSetRenderer()._render_rows(dataset, columns, maxlen)
271
shorter
272
little longer b
273
"""
274
for row in dataset:
275
print self.separator.join([
276
"{0!s:{1}}".format(row[column], maxlen[column])
277
for column in columns])
278
279
def _render_dataset(self, dataset):
280
"""
281
Render the header followed by the rows of data.
282
"""
283
dataset, columns, maxlen = self._analyze_dataset(dataset)
284
self._render_header(dataset, columns, maxlen)
285
self._render_rows(dataset, columns, maxlen)
286
287
def _render_empty_dataset(self):
288
"""
289
Render empty dataset.
290
291
By default it just prints out a fixed sentence:
292
>>> DataSetRenderer()._render_empty_dataset()
293
There is no data to display
294
295
This can be changed by passing an argument to the constructor
296
>>> DataSetRenderer(empty="there is no data")._render_empty_dataset()
297
there is no data
298
"""
299
print self.empty
300
301
def render(self, dataset):
302
if len(dataset) > 0:
303
self._render_dataset(dataset)
304
else:
305
self._render_empty_dataset()
306
307
308
class XMLRPCCommand(Command):
309
"""
310
Abstract base class for commands that interact with dashboard server
311
over XML-RPC.
312
313
The only difference is that you should implement invoke_remote()
314
instead of invoke(). The provided implementation catches several
315
socket and XML-RPC errors and prints a pretty error message.
316
"""
317
318
@staticmethod
319
def _construct_xml_rpc_url(url):
320
"""
321
Construct URL to the XML-RPC service out of the given URL
322
"""
323
parts = urlparse.urlsplit(url)
324
if not parts.path.endswith("/RPC2/"):
325
path = parts.path.rstrip("/") + "/xml-rpc/"
326
else:
327
path = parts.path
328
return urlparse.urlunsplit(
329
(parts.scheme, parts.netloc, path, "", ""))
330
331
@staticmethod
332
def _strict_server_version(version):
333
"""
334
Calculate strict server version (as defined by
335
distutils.version.StrictVersion). This works by discarding .candidate
336
and .dev release-levels.
337
>>> XMLRPCCommand._strict_server_version("0.4.0.candidate.5")
338
'0.4.0'
339
>>> XMLRPCCommand._strict_server_version("0.4.0.dev.126")
340
'0.4.0'
341
>>> XMLRPCCommand._strict_server_version("0.4.0.alpha.1")
342
'0.4.0a1'
343
>>> XMLRPCCommand._strict_server_version("0.4.0.beta.2")
344
'0.4.0b2'
345
"""
346
try:
347
major, minor, micro, releaselevel, serial = version.split(".")
348
except ValueError:
349
raise ValueError(
350
("version %r does not follow pattern "
351
"'major.minor.micro.releaselevel.serial'") % version)
352
if releaselevel in ["dev", "candidate", "final"]:
353
return "%s.%s.%s" % (major, minor, micro)
354
elif releaselevel == "alpha":
355
return "%s.%s.%sa%s" % (major, minor, micro, serial)
356
elif releaselevel == "beta":
357
return "%s.%s.%sb%s" % (major, minor, micro, serial)
358
else:
359
raise ValueError(
360
("releaselevel %r is not one of 'final', 'alpha', 'beta', "
361
"'candidate' or 'final'") % releaselevel)
362
363
def _check_server_version(self, server_obj, required_version):
364
"""
365
Check that server object has is at least required_version.
366
367
This method may raise InsufficientServerVersion.
368
"""
369
from distutils.version import StrictVersion, LooseVersion
370
# For backwards compatibility the server reports
371
# major.minor.micro.releaselevel.serial which is not PEP-386 compliant
372
server_version = StrictVersion(
373
self._strict_server_version(server_obj.version()))
374
required_version = StrictVersion(required_version)
375
if server_version < required_version:
376
raise InsufficientServerVersion(server_version, required_version)
377
378
def __init__(self, parser, args):
379
super(XMLRPCCommand, self).__init__(parser, args)
380
xml_rpc_url = self._construct_xml_rpc_url(self.args.dashboard_url)
381
self.server = AuthenticatingServerProxy(
382
xml_rpc_url,
383
verbose=args.verbose_xml_rpc,
384
allow_none=True,
385
use_datetime=True,
386
auth_backend=KeyringAuthBackend())
387
388
def use_non_legacy_api_if_possible(self, name='server'):
389
# Legacy APIs are registered in top-level object, non-legacy APIs are
390
# prefixed with extension name.
391
if "dashboard.version" in getattr(self, name).system.listMethods():
392
setattr(self, name, getattr(self, name).dashboard)
393
394
@classmethod
395
def register_arguments(cls, parser):
396
dashboard_group = parser.add_argument_group("dashboard specific arguments")
397
default_dashboard_url = os.getenv("DASHBOARD_URL")
398
if default_dashboard_url:
399
dashboard_group.add_argument("--dashboard-url",
400
metavar="URL", help="URL of your validation dashboard (currently %(default)s)",
401
default=default_dashboard_url)
402
else:
403
dashboard_group.add_argument("--dashboard-url", required=True,
404
metavar="URL", help="URL of your validation dashboard")
405
debug_group = parser.add_argument_group("debugging arguments")
406
debug_group.add_argument("--verbose-xml-rpc",
407
action="store_true", default=False,
408
help="Show XML-RPC data")
409
return dashboard_group
410
411
@contextlib.contextmanager
412
def safety_net(self):
413
try:
414
yield
415
except socket.error as ex:
416
print >> sys.stderr, "Unable to connect to server at %s" % (
417
self.args.dashboard_url,)
418
# It seems that some errors are reported as -errno
419
# while others as +errno.
420
ex.errno = abs(ex.errno)
421
if ex.errno == errno.ECONNREFUSED:
422
print >> sys.stderr, "Connection was refused."
423
parts = urlparse.urlsplit(self.args.dashboard_url)
424
if parts.netloc == "localhost:8000":
425
print >> sys.stderr, "Perhaps the server is not running?"
426
elif ex.errno == errno.ENOENT:
427
print >> sys.stderr, "Unable to resolve address"
428
else:
429
print >> sys.stderr, "Socket %d: %s" % (ex.errno, ex.strerror)
430
except xmlrpclib.ProtocolError as ex:
431
print >> sys.stderr, "Unable to exchange XML-RPC message with dashboard server"
432
print >> sys.stderr, "HTTP error code: %d/%s" % (ex.errcode, ex.errmsg)
433
except xmlrpclib.Fault as ex:
434
self.handle_xmlrpc_fault(ex.faultCode, ex.faultString)
435
except InsufficientServerVersion as ex:
436
print >> sys.stderr, ("This command requires at least server version "
437
"%s, actual server version is %s" %
438
(ex.required_version, ex.server_version))
439
440
def invoke(self):
441
with self.safety_net():
442
self.use_non_legacy_api_if_possible()
443
return self.invoke_remote()
444
445
def handle_xmlrpc_fault(self, faultCode, faultString):
446
if faultCode == 500:
447
print >> sys.stderr, "Dashboard server has experienced internal error"
448
print >> sys.stderr, faultString
449
else:
450
print >> sys.stderr, "XML-RPC error %d: %s" % (faultCode, faultString)
451
452
def invoke_remote(self):
453
raise NotImplementedError()
454
455
456
class server_version(XMLRPCCommand):
457
"""
458
Display dashboard server version
459
"""
460
461
def invoke_remote(self):
462
print "Dashboard server version: %s" % (self.server.version(),)
463
464
465
class put(XMLRPCCommand):
466
"""
467
Upload a bundle on the server
468
"""
469
470
@classmethod
471
def register_arguments(cls, parser):
472
super(put, cls).register_arguments(parser)
473
parser.add_argument("LOCAL",
474
type=argparse.FileType("rb"),
475
help="pathname on the local file system")
476
parser.add_argument("REMOTE",
477
default="/anonymous/", nargs='?',
478
help="pathname on the server")
479
480
def invoke_remote(self):
481
content = self.args.LOCAL.read()
482
filename = self.args.LOCAL.name
483
pathname = self.args.REMOTE
484
content_sha1 = self.server.put(content, filename, pathname)
485
print "Stored as bundle {0}".format(content_sha1)
486
487
def handle_xmlrpc_fault(self, faultCode, faultString):
488
if faultCode == 404:
489
print >> sys.stderr, "Bundle stream %s does not exist" % (
490
self.args.REMOTE)
491
elif faultCode == 409:
492
print >> sys.stderr, "You have already uploaded this bundle to the dashboard"
493
else:
494
super(put, self).handle_xmlrpc_fault(faultCode, faultString)
495
496
497
class get(XMLRPCCommand):
498
"""
499
Download a bundle from the server
500
"""
501
502
@classmethod
503
def register_arguments(cls, parser):
504
super(get, cls).register_arguments(parser)
505
parser.add_argument("SHA1",
506
type=str,
507
help="SHA1 of the bundle to download")
508
parser.add_argument("--overwrite",
509
action="store_true",
510
help="Overwrite files on the local disk")
511
parser.add_argument("--output", "-o",
512
type=argparse.FileType("wb"),
513
default=None,
514
help="Alternate name of the output file")
515
516
def invoke_remote(self):
517
response = self.server.get(self.args.SHA1)
518
if self.args.output is None:
519
filename = self.args.SHA1
520
if os.path.exists(filename) and not self.args.overwrite:
521
print >> sys.stderr, "File {filename!r} already exists".format(
522
filename=filename)
523
print >> sys.stderr, "You may pass --overwrite to write over it"
524
return -1
525
stream = open(filename, "wb")
526
else:
527
stream = self.args.output
528
filename = self.args.output.name
529
stream.write(response['content'])
530
print "Downloaded bundle {0} to file {1!r}".format(
531
self.args.SHA1, filename)
532
533
def handle_xmlrpc_fault(self, faultCode, faultString):
534
if faultCode == 404:
535
print >> sys.stderr, "Bundle {sha1} does not exist".format(
536
sha1=self.args.SHA1)
537
else:
538
super(get, self).handle_xmlrpc_fault(faultCode, faultString)
539
540
541
class deserialize(XMLRPCCommand):
542
"""
543
Deserialize a bundle on the server
544
"""
545
546
@classmethod
547
def register_arguments(cls, parser):
548
super(deserialize, cls).register_arguments(parser)
549
parser.add_argument("SHA1",
550
type=str,
551
help="SHA1 of the bundle to deserialize")
552
553
def invoke_remote(self):
554
response = self.server.deserialize(self.args.SHA1)
555
print "Bundle {sha1} deserialized".format(
556
sha1=self.args.SHA1)
557
558
def handle_xmlrpc_fault(self, faultCode, faultString):
559
if faultCode == 404:
560
print >> sys.stderr, "Bundle {sha1} does not exist".format(
561
sha1=self.args.SHA1)
562
elif faultCode == 409:
563
print >> sys.stderr, "Unable to deserialize bundle {sha1}".format(
564
sha1=self.args.SHA1)
565
print >> sys.stderr, faultString
566
else:
567
super(deserialize, self).handle_xmlrpc_fault(faultCode, faultString)
568
569
570
def _get_pretty_renderer(**kwargs):
571
if "separator" not in kwargs:
572
kwargs["separator"] = " | "
573
if "header_separator" not in kwargs:
574
kwargs["header_separator"] = True
575
return DataSetRenderer(**kwargs)
576
577
578
class streams(XMLRPCCommand):
579
"""
580
Show streams you have access to
581
"""
582
583
renderer = _get_pretty_renderer(
584
order=('pathname', 'bundle_count', 'name'),
585
column_map={
586
'pathname': 'Pathname',
587
'bundle_count': 'Number of bundles',
588
'name': 'Name'},
589
row_formatter={
590
'name': lambda name: name or "(not set)"},
591
empty="There are no streams you can access on the server",
592
caption="Bundle streams")
593
594
def invoke_remote(self):
595
self.renderer.render(self.server.streams())
596
597
598
class bundles(XMLRPCCommand):
599
"""
600
Show bundles in the specified stream
601
"""
602
603
renderer = _get_pretty_renderer(
604
column_map={
605
'uploaded_by': 'Uploader',
606
'uploaded_on': 'Upload date',
607
'content_filename': 'File name',
608
'content_sha1': 'SHA1',
609
'is_deserialized': "Deserialized?"},
610
row_formatter={
611
'is_deserialized': lambda x: "yes" if x else "no",
612
'uploaded_by': lambda x: x or "(anonymous)",
613
'uploaded_on': lambda x: x.strftime("%Y-%m-%d %H:%M:%S")},
614
order=('content_sha1', 'content_filename', 'uploaded_by',
615
'uploaded_on', 'is_deserialized'),
616
empty="There are no bundles in this stream",
617
caption="Bundles",
618
separator=" | ")
619
620
@classmethod
621
def register_arguments(cls, parser):
622
super(bundles, cls).register_arguments(parser)
623
parser.add_argument("PATHNAME",
624
default="/anonymous/", nargs='?',
625
help="pathname on the server (defaults to %(default)s)")
626
627
def invoke_remote(self):
628
self.renderer.render(self.server.bundles(self.args.PATHNAME))
629
630
def handle_xmlrpc_fault(self, faultCode, faultString):
631
if faultCode == 404:
632
print >> sys.stderr, "Bundle stream %s does not exist" % (
633
self.args.PATHNAME)
634
else:
635
super(bundles, self).handle_xmlrpc_fault(faultCode, faultString)
636
637
638
class make_stream(XMLRPCCommand):
639
"""
640
Create a bundle stream on the server
641
"""
642
643
@classmethod
644
def register_arguments(cls, parser):
645
super(make_stream, cls).register_arguments(parser)
646
parser.add_argument(
647
"pathname",
648
type=str,
649
help="Pathname of the bundle stream to create")
650
parser.add_argument(
651
"--name",
652
type=str,
653
default="",
654
help="Name of the bundle stream (description)")
655
656
def invoke_remote(self):
657
self._check_server_version(self.server, "0.3")
658
pathname = self.server.make_stream(self.args.pathname, self.args.name)
659
print "Bundle stream {pathname} created".format(pathname=pathname)
660
661
662
class backup(XMLRPCCommand):
663
"""
664
Backup data uploaded to a dashboard instance.
665
666
Not all data is preserved. The following data is lost: identity of the user
667
that uploaded each bundle, time of uploading and deserialization on the
668
server, name of the bundle stream that contained the data
669
"""
670
671
@classmethod
672
def register_arguments(cls, parser):
673
super(backup, cls).register_arguments(parser)
674
parser.add_argument("BACKUP_DIR", type=str,
675
help="Directory to backup to")
676
677
def invoke_remote(self):
678
if not os.path.exists(self.args.BACKUP_DIR):
679
os.mkdir(self.args.BACKUP_DIR)
680
for bundle_stream in self.server.streams():
681
print "Processing stream %s" % bundle_stream["pathname"]
682
bundle_stream_dir = os.path.join(self.args.BACKUP_DIR, urllib.quote_plus(bundle_stream["pathname"]))
683
if not os.path.exists(bundle_stream_dir):
684
os.mkdir(bundle_stream_dir)
685
with open(os.path.join(bundle_stream_dir, "metadata.json"), "wt") as stream:
686
simplejson.dump({
687
"pathname": bundle_stream["pathname"],
688
"name": bundle_stream["name"],
689
"user": bundle_stream["user"],
690
"group": bundle_stream["group"],
691
}, stream)
692
for bundle in self.server.bundles(bundle_stream["pathname"]):
693
print " * Backing up bundle %s" % bundle["content_sha1"]
694
data = self.server.get(bundle["content_sha1"])
695
bundle_pathname = os.path.join(bundle_stream_dir, bundle["content_sha1"])
696
# Note: we write bundles as binary data to preserve anything the user might have dumped on us
697
with open(bundle_pathname + ".json", "wb") as stream:
698
stream.write(data["content"])
699
with open(bundle_pathname + ".metadata.json", "wt") as stream:
700
simplejson.dump({
701
"uploaded_by": bundle["uploaded_by"],
702
"uploaded_on": datetime_extension.to_json(bundle["uploaded_on"]),
703
"content_filename": bundle["content_filename"],
704
"content_sha1": bundle["content_sha1"],
705
"content_size": bundle["content_size"],
706
}, stream)
707
708
709
class restore(XMLRPCCommand):
710
"""
711
Restore a dashboard instance from backup
712
"""
713
714
@classmethod
715
def register_arguments(cls, parser):
716
super(restore, cls).register_arguments(parser)
717
parser.add_argument("BACKUP_DIR", type=str,
718
help="Directory to backup from")
719
720
def invoke_remote(self):
721
self._check_server_version(self.server, "0.3")
722
for stream_pathname_quoted in os.listdir(self.args.BACKUP_DIR):
723
filesystem_stream_pathname = os.path.join(self.args.BACKUP_DIR, stream_pathname_quoted)
724
if not os.path.isdir(filesystem_stream_pathname):
725
continue
726
stream_pathname = urllib.unquote(stream_pathname_quoted)
727
if os.path.exists(os.path.join(filesystem_stream_pathname, "metadata.json")):
728
with open(os.path.join(filesystem_stream_pathname, "metadata.json"), "rt") as stream:
729
stream_metadata = simplejson.load(stream)
730
else:
731
stream_metadata = {}
732
print "Processing stream %s" % stream_pathname
733
try:
734
self.server.make_stream(stream_pathname, stream_metadata.get("name", "Restored from backup"))
735
except xmlrpclib.Fault as ex:
736
if ex.faultCode != 409:
737
raise
738
for content_sha1 in [item[:-len(".json")] for item in os.listdir(filesystem_stream_pathname) if item.endswith(".json") and not item.endswith(".metadata.json") and item != "metadata.json"]:
739
filesystem_content_filename = os.path.join(filesystem_stream_pathname, content_sha1 + ".json")
740
if not os.path.isfile(filesystem_content_filename):
741
continue
742
with open(os.path.join(filesystem_stream_pathname, content_sha1) + ".metadata.json", "rt") as stream:
743
bundle_metadata = simplejson.load(stream)
744
with open(filesystem_content_filename, "rb") as stream:
745
content = stream.read()
746
print " * Restoring bundle %s" % content_sha1
747
try:
748
self.server.put(content, bundle_metadata["content_filename"], stream_pathname)
749
except xmlrpclib.Fault as ex:
750
if ex.faultCode != 409:
751
raise
752
753
754
class pull(ExperimentalCommandMixIn, XMLRPCCommand):
755
"""
756
Copy bundles and bundle streams from one dashboard to another.
757
758
This command checks for two environment varialbes:
759
The value of DASHBOARD_URL is used as a replacement for --dashbard-url.
760
The value of REMOTE_DASHBOARD_URL as a replacement for FROM.
761
Their presence automatically makes the corresponding argument optional.
762
"""
763
764
def __init__(self, parser, args):
765
super(pull, self).__init__(parser, args)
766
remote_xml_rpc_url = self._construct_xml_rpc_url(self.args.FROM)
767
self.remote_server = AuthenticatingServerProxy(
768
remote_xml_rpc_url,
769
verbose=args.verbose_xml_rpc,
770
use_datetime=True,
771
allow_none=True,
772
auth_backend=KeyringAuthBackend())
773
self.use_non_legacy_api_if_possible('remote_server')
774
775
@classmethod
776
def register_arguments(cls, parser):
777
group = super(pull, cls).register_arguments(parser)
778
default_remote_dashboard_url = os.getenv("REMOTE_DASHBOARD_URL")
779
if default_remote_dashboard_url:
780
group.add_argument(
781
"FROM", nargs="?",
782
help="URL of the remote validation dashboard (currently %(default)s)",
783
default=default_remote_dashboard_url)
784
else:
785
group.add_argument(
786
"FROM",
787
help="URL of the remote validation dashboard)")
788
group.add_argument("STREAM", nargs="*", help="Streams to pull from (all by default)")
789
790
@staticmethod
791
def _filesizeformat(num_bytes):
792
"""
793
Formats the value like a 'human-readable' file size (i.e. 13 KB, 4.1 MB,
794
102 num_bytes, etc).
795
"""
796
try:
797
num_bytes = float(num_bytes)
798
except (TypeError, ValueError, UnicodeDecodeError):
799
return "%(size)d byte", "%(size)d num_bytes" % {'size': 0}
800
801
filesize_number_format = lambda value: "%0.2f" % (round(value, 1),)
802
803
if num_bytes < 1024:
804
return "%(size)d bytes" % {'size': num_bytes}
805
if num_bytes < 1024 * 1024:
806
return "%s KB" % filesize_number_format(num_bytes / 1024)
807
if num_bytes < 1024 * 1024 * 1024:
808
return "%s MB" % filesize_number_format(num_bytes / (1024 * 1024))
809
return "%s GB" % filesize_number_format(num_bytes / (1024 * 1024 * 1024))
810
811
def invoke_remote(self):
812
self._check_server_version(self.server, "0.3")
813
814
print "Checking local and remote streams"
815
remote = self.remote_server.streams()
816
if self.args.STREAM:
817
# Check that all requested streams are available remotely
818
requested_set = frozenset(self.args.STREAM)
819
remote_set = frozenset((stream["pathname"] for stream in remote))
820
unavailable_set = requested_set - remote_set
821
if unavailable_set:
822
print >> sys.stderr, "Remote stream not found: %s" % ", ".join(unavailable_set)
823
return -1
824
# Limit to requested streams if necessary
825
remote = [stream for stream in remote if stream["pathname"] in requested_set]
826
local = self.server.streams()
827
missing_pathnames = set([stream["pathname"] for stream in remote]) - set([stream["pathname"] for stream in local])
828
for stream in remote:
829
if stream["pathname"] in missing_pathnames:
830
self.server.make_stream(stream["pathname"], stream["name"])
831
local_bundles = []
832
else:
833
local_bundles = [bundle for bundle in self.server.bundles(stream["pathname"])]
834
remote_bundles = [bundle for bundle in self.remote_server.bundles(stream["pathname"])]
835
missing_bundles = set((bundle["content_sha1"] for bundle in remote_bundles))
836
missing_bundles -= set((bundle["content_sha1"] for bundle in local_bundles))
837
try:
838
missing_bytes = sum(
839
(bundle["content_size"]
840
for bundle in remote_bundles
841
if bundle["content_sha1"] in missing_bundles))
842
except KeyError as ex:
843
# Older servers did not return content_size so this part is optional
844
missing_bytes = None
845
if missing_bytes:
846
print "Stream %s needs update (%s)" % (stream["pathname"], self._filesizeformat(missing_bytes))
847
elif missing_bundles:
848
print "Stream %s needs update (no estimate available)" % (stream["pathname"],)
849
else:
850
print "Stream %s is up to date" % (stream["pathname"],)
851
for content_sha1 in missing_bundles:
852
print "Getting %s" % (content_sha1,),
853
sys.stdout.flush()
854
data = self.remote_server.get(content_sha1)
855
print "got %s, storing" % (self._filesizeformat(len(data["content"]))),
856
sys.stdout.flush()
857
try:
858
self.server.put(data["content"], data["content_filename"], stream["pathname"])
859
except xmlrpclib.Fault as ex:
860
if ex.faultCode == 409: # duplicate
861
print "already present (in another stream)"
862
else:
863
raise
864
else:
865
print "done"
866
867
868
class data_views(ExperimentalCommandMixIn, XMLRPCCommand):
869
"""
870
Show data views defined on the server
871
"""
872
renderer = _get_pretty_renderer(
873
column_map={
874
'name': 'Name',
875
'summary': 'Summary',
876
},
877
order=('name', 'summary'),
878
empty="There are no data views defined yet",
879
caption="Data Views")
880
881
def invoke_remote(self):
882
self._check_server_version(self.server, "0.4")
883
self.renderer.render(self.server.data_views())
884
print
885
print "Tip: to invoke a data view try `lc-tool query-data-view`"
886
887
888
class query_data_view(ExperimentalCommandMixIn, XMLRPCCommand):
889
"""
890
Invoke a specified data view
891
"""
892
@classmethod
893
def register_arguments(cls, parser):
894
super(query_data_view, cls).register_arguments(parser)
895
parser.add_argument("QUERY", metavar="QUERY", nargs="...",
896
help="Data view name and any optional and required arguments")
897
898
def _probe_data_views(self):
899
"""
900
Probe the server for information about data views
901
"""
902
with self.safety_net():
903
self.use_non_legacy_api_if_possible()
904
self._check_server_version(self.server, "0.4")
905
return self.server.data_views()
906
907
def reparse_arguments(self, parser, raw_args):
908
self.data_views = self._probe_data_views()
909
if self.data_views is None:
910
return
911
# Here we hack a little, the last actuin is the QUERY action added
912
# in register_arguments above. By removing it we make the output
913
# of lc-tool query-data-view NAME --help more consistent.
914
del parser._actions[-1]
915
subparsers = parser.add_subparsers(
916
title="Data views available on the server")
917
for data_view in self.data_views:
918
data_view_parser = subparsers.add_parser(
919
data_view["name"],
920
help=data_view["summary"],
921
epilog=data_view["documentation"])
922
data_view_parser.set_defaults(data_view=data_view)
923
group = data_view_parser.add_argument_group("Data view parameters")
924
for argument in data_view["arguments"]:
925
if argument["default"] is None:
926
group.add_argument(
927
"--{name}".format(name=argument["name"].replace("_", "-")),
928
dest=argument["name"],
929
help=argument["help"],
930
type=str,
931
required=True)
932
else:
933
group.add_argument(
934
"--{name}".format(name=argument["name"].replace("_", "-")),
935
dest=argument["name"],
936
help=argument["help"],
937
type=str,
938
default=argument["default"])
939
self.args = self.parser.parse_args(raw_args)
940
941
def invoke(self):
942
# Override and _not_ call 'use_non_legacy_api_if_possible' as we
943
# already did this reparse_arguments
944
with self.safety_net():
945
return self.invoke_remote()
946
947
def invoke_remote(self):
948
if self.data_views is None:
949
return -1
950
self._check_server_version(self.server, "0.4")
951
# Build a collection of arguments for data view
952
data_view_args = {}
953
for argument in self.args.data_view["arguments"]:
954
arg_name = argument["name"]
955
if arg_name in self.args:
956
data_view_args[arg_name] = getattr(self.args, arg_name)
957
# Invoke the data view
958
response = self.server.query_data_view(self.args.data_view["name"], data_view_args)
959
# Create a pretty-printer
960
renderer = _get_pretty_renderer(
961
caption=self.args.data_view["summary"],
962
order=[item["name"] for item in response["columns"]])
963
# Post-process the data so that it fits the printer
964
data_for_renderer = [
965
dict(zip(
966
[column["name"] for column in response["columns"]],
967
row))
968
for row in response["rows"]]
969
# Print the data
970
renderer.render(data_for_renderer)
971
972
973
class version(Command):
974
"""
975
Show dashboard client version
976
"""
977
def invoke(self):
978
import versiontools
979
from lava_dashboard_tool import __version__
980
print "Dashboard client version: {version}".format(
981
version=versiontools.format_version(__version__))
Loggerhead is a web-based interface for Breezy
Version: 2.0.1