1
# Copyright (C) 2010 Linaro Limited
3
# Author: Zygmunt Krynicki <zygmunt.krynicki@linaro.org>
5
# This file is part of LAVA Dashboard
7
# Launch Control is free software: you can redistribute it and/or modify
8
# it under the terms of the GNU Affero General Public License version 3
9
# as published by the Free Software Foundation
11
# Launch Control is distributed in the hope that it will be useful,
12
# but WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14
# GNU General Public License for more details.
16
# You should have received a copy of the GNU Affero General Public License
17
# along with Launch Control. If not, see <http://www.gnu.org/licenses/>.
31
from django.contrib.auth.models import User, Group
32
from django.core.urlresolvers import reverse
33
from django.db import IntegrityError, DatabaseError
34
from linaro_django_xmlrpc.models import (
40
from dashboard_app.filters import evaluate_filter
41
from dashboard_app.models import (
48
from lava_scheduler_app.models import (
55
A namespace for error codes that may be returned by various XML-RPC
56
methods. Where applicable existing status codes from HTTP protocol
66
INTERNAL_SERVER_ERROR = 500
70
class DashboardAPI(ExposedAPI):
74
All public methods are automatically exposed as XML-RPC methods
77
@xml_rpc_signature('str')
86
Return dashboard server version. The version is a string with
87
dots separating five components.
96
See: http://docs.python.org/library/sys.html#sys.version_info
98
Note that this version will change to reflect the new versioning
99
scheme, based on git tags named after release dates instead of
100
arbitrary major and minor versions, once the migration to packaging
105
Server version string
107
return ".".join(map(str, (0, 29, 0, "final", 0)))
109
def _put(self, content, content_filename, pathname):
111
logging.debug("Getting bundle stream")
112
if self.user and self.user.is_superuser:
113
bundle_stream = BundleStream.objects.get(pathname=pathname)
115
bundle_stream = BundleStream.objects.accessible_by_principal(self.user).get(pathname=pathname)
116
except BundleStream.DoesNotExist:
117
logging.debug("Bundle stream does not exist, aborting")
118
raise xmlrpclib.Fault(errors.NOT_FOUND,
119
"Bundle stream not found")
120
if not bundle_stream.can_upload(self.user):
121
raise xmlrpclib.Fault(
122
errors.FORBIDDEN, "You cannot upload to this stream")
124
logging.debug("Creating bundle object")
125
bundle = Bundle.objects.create_with_content(bundle_stream, self.user, content_filename, content)
126
except (IntegrityError, ValueError) as exc:
127
logging.debug("Raising xmlrpclib.Fault(errors.CONFLICT)")
128
raise xmlrpclib.Fault(errors.CONFLICT, str(exc))
130
logging.exception("big oops")
133
logging.debug("Deserializing bundle")
137
@xml_rpc_signature('str', 'str', 'str', 'str')
138
def put(self, content, content_filename, pathname):
142
`put` (`content`, `content_filename`, `pathname`)
146
Upload a bundle to the server.
151
Full text of the bundle. This *SHOULD* be a valid JSON
152
document and it *SHOULD* match the "Dashboard Bundle Format
153
1.0" schema. The SHA1 of the content *MUST* be unique or a
154
``Fault(409, "...")`` is raised. This is used to protect
155
from simple duplicate submissions.
156
`content_filename`: string
157
Name of the file that contained the text of the bundle. The
158
`content_filename` can be an arbitrary string and will be
159
stored along with the content for reference.
161
Pathname of the bundle stream where a new bundle should
162
be created and stored. This argument *MUST* designate a
163
pre-existing bundle stream or a ``Fault(404, "...")`` exception
164
is raised. In addition the user *MUST* have access
165
permission to upload bundles there or a ``Fault(403, "...")``
166
exception is raised. See below for access rules.
170
If all goes well this function returns the SHA1 of the content.
177
- Bundle stream not found
178
- Uploading to specified stream is not permitted
180
Duplicate bundle content
182
Rules for bundle stream access
183
------------------------------
184
The following rules govern bundle stream upload access rights:
185
- all anonymous streams are accessible
186
- personal streams are accessible to owners
187
- team streams are accessible to team members
190
bundle = self._put(content, content_filename, pathname)
191
logging.debug("Returning bundle SHA1")
192
return bundle.content_sha1
194
@xml_rpc_signature('str', 'str', 'str', 'str')
195
def put_ex(self, content, content_filename, pathname):
199
`put` (`content`, `content_filename`, `pathname`)
203
Upload a bundle to the server. A variant on put_ex that returns the
204
URL of the bundle instead of its SHA1.
209
Full text of the bundle. This *SHOULD* be a valid JSON
210
document and it *SHOULD* match the "Dashboard Bundle Format
211
1.0" schema. The SHA1 of the content *MUST* be unique or a
212
``Fault(409, "...")`` is raised. This is used to protect
213
from simple duplicate submissions.
214
`content_filename`: string
215
Name of the file that contained the text of the bundle. The
216
`content_filename` can be an arbitrary string and will be
217
stored along with the content for reference.
219
Pathname of the bundle stream where a new bundle should
220
be created and stored. This argument *MUST* designate a
221
pre-existing bundle stream or a ``Fault(404, "...")`` exception
222
is raised. In addition the user *MUST* have access
223
permission to upload bundles there or a ``Fault(403, "...")``
224
exception is raised. See below for access rules.
228
If all goes well this function returns the full URL of the bundle.
235
- Bundle stream not found
236
- Uploading to specified stream is not permitted
238
Duplicate bundle content
240
Rules for bundle stream access
241
------------------------------
242
The following rules govern bundle stream upload access rights:
243
- all anonymous streams are accessible
244
- personal streams are accessible to owners
245
- team streams are accessible to team members
248
bundle = self._put(content, content_filename, pathname)
249
logging.debug("Returning permalink to bundle")
250
return self._context.request.build_absolute_uri(
251
reverse('dashboard_app.views.redirect_to_bundle',
252
kwargs={'content_sha1': bundle.content_sha1}))
254
def put_pending(self, content, pathname, group_name):
258
`put_pending` (`content`, `pathname`, `group_name`)
262
MultiNode internal call.
264
Stores the bundle until the coordinator allows the complete
265
bundle list to be aggregated from the list and submitted by put_group
270
Full text of the bundle. This *MUST* be a valid JSON
271
document and it *SHOULD* match the "Dashboard Bundle Format
272
1.0" schema. The SHA1 of the content *MUST* be unique or a
273
``Fault(409, "...")`` is raised. This is used to protect
274
from simple duplicate submissions.
276
Pathname of the bundle stream where a new bundle should
277
be created and stored. This argument *MUST* designate a
278
pre-existing bundle stream or a ``Fault(404, "...")`` exception
279
is raised. In addition the user *MUST* have access
280
permission to upload bundles there or a ``Fault(403, "...")``
281
exception is raised. See below for access rules.
283
Unique ID of the MultiNode group. Other pending bundles will
284
be aggregated into a single result bundle for this group.
288
If all goes well this function returns the SHA1 of the content.
295
- Bundle stream not found
296
- Uploading to specified stream is not permitted
298
Duplicate bundle content
300
Rules for bundle stream access
301
------------------------------
302
The following rules govern bundle stream upload access rights:
303
- all anonymous streams are accessible
304
- personal streams are accessible to owners
305
- team streams are accessible to team members
309
logging.debug("Getting bundle stream")
310
if self.user.is_superuser:
311
bundle_stream = BundleStream.objects.get(pathname=pathname)
313
bundle_stream = BundleStream.objects.accessible_by_principal(self.user).get(pathname=pathname)
314
except BundleStream.DoesNotExist:
315
logging.debug("Bundle stream does not exist, aborting")
316
raise xmlrpclib.Fault(errors.NOT_FOUND,
317
"Bundle stream not found")
318
if not bundle_stream.can_upload(self.user):
319
raise xmlrpclib.Fault(
320
errors.FORBIDDEN, "You cannot upload to this stream")
322
# add this to a list which put_group can use.
323
sha1 = hashlib.sha1()
325
hexdigest = sha1.hexdigest()
326
groupfile = "/tmp/%s" % group_name
327
with open(groupfile, "a+") as grp_file:
328
grp_file.write("%s\n" % content)
330
except Exception as e:
331
logging.debug("Dashboard pending submission caused an exception: %s" % e)
333
def put_group(self, content, content_filename, pathname, group_name):
337
`put_group` (`content`, `content_filename`, `pathname`, `group_name`)
341
MultiNode internal call.
343
Adds the final bundle to the list, aggregates the list
344
into a single group bundle and submits the group bundle.
349
Full text of the bundle. This *MUST* be a valid JSON
350
document and it *SHOULD* match the "Dashboard Bundle Format
351
1.0" schema. The SHA1 of the content *MUST* be unique or a
352
``Fault(409, "...")`` is raised. This is used to protect
353
from simple duplicate submissions.
354
`content_filename`: string
355
Name of the file that contained the text of the bundle. The
356
`content_filename` can be an arbitrary string and will be
357
stored along with the content for reference.
359
Pathname of the bundle stream where a new bundle should
360
be created and stored. This argument *MUST* designate a
361
pre-existing bundle stream or a ``Fault(404, "...")`` exception
362
is raised. In addition the user *MUST* have access
363
permission to upload bundles there or a ``Fault(403, "...")``
364
exception is raised. See below for access rules.
366
Unique ID of the MultiNode group. Other pending bundles will
367
be aggregated into a single result bundle for this group. At
368
least one other bundle must have already been submitted as
369
pending for the specified MultiNode group. LAVA Coordinator
370
causes the parent job to wait until all nodes have been marked
371
as having pending bundles, even if some bundles are empty.
375
If all goes well this function returns the full URL of the bundle.
380
One or more bundles could not be converted to JSON prior
385
- Bundle stream not found
386
- Uploading to specified stream is not permitted
388
Duplicate bundle content
390
Rules for bundle stream access
391
------------------------------
392
The following rules govern bundle stream upload access rights:
393
- all anonymous streams are accessible
394
- personal streams are accessible to owners
395
- team streams are accessible to team members
398
grp_file = "/tmp/%s" % group_name
400
bundle_set[group_name] = []
401
if os.path.isfile(grp_file):
402
with open(grp_file, "r") as grp_data:
403
grp_list = grp_data.readlines()
404
for testrun in grp_list:
405
bundle_set[group_name].append(json.loads(testrun))
406
# Note: now that we have the data from the group, the group data file could be re-used
407
# as an error log which is simpler than debugging through XMLRPC.
409
raise ValueError("Aggregation failure for %s - check coordinator rpc_delay?" % group_name)
412
json_data = json.loads(content)
414
logging.debug("Invalid JSON content within the sub_id zero bundle")
417
bundle_set[group_name].append(json_data)
418
except Exception as e:
419
logging.debug("appending JSON caused exception %s" % e)
421
for bundle_list in bundle_set[group_name]:
422
for test_run in bundle_list['test_runs']:
423
group_tests.append(test_run)
424
except Exception as e:
425
logging.debug("aggregating bundles caused exception %s" % e)
426
group_content = json.dumps({"test_runs": group_tests, "format": json_data['format']})
427
bundle = self._put(group_content, content_filename, pathname)
428
logging.debug("Returning permalink to aggregated bundle for %s" % group_name)
429
permalink = self._context.request.build_absolute_uri(
430
reverse('dashboard_app.views.redirect_to_bundle',
431
kwargs={'content_sha1': bundle.content_sha1}))
432
# only delete the group file when things go well.
433
if os.path.isfile(grp_file):
437
def get(self, content_sha1):
441
`get` (`content_sha1`)
445
Download a bundle from the server.
449
`content_sha1`: string
450
SHA1 hash of the content of the bundle to download. This
451
*MUST* designate an bundle or ``Fault(404, "...")`` is raised.
455
This function returns an XML-RPC struct with the following fields:
457
`content_filename`: string
458
The value that was stored on a previous call to put()
460
The full text of the bundle
464
- 404 Bundle not found
465
- 403 Permission denied
467
Rules for bundle stream access
468
------------------------------
469
The following rules govern bundle stream download access rights:
470
- all anonymous streams are accessible
471
- personal streams are accessible to owners
472
- team streams are accessible to team members
475
bundle = Bundle.objects.get(content_sha1=content_sha1)
476
if not bundle.bundle_stream.is_accessible_by(self.user):
477
raise xmlrpclib.Fault(
478
403, "Permission denied. User does not have permissions "
479
"to access this bundle.")
480
except Bundle.DoesNotExist:
481
raise xmlrpclib.Fault(errors.NOT_FOUND,
484
return {"content": bundle.content.read(),
485
"content_filename": bundle.content_filename}
487
@xml_rpc_signature('struct')
496
List all bundle streams that the user has access to
504
This function returns an XML-RPC array of XML-RPC structs with
505
the following fields:
508
The pathname of the bundle stream
510
The user-configurable name of the bundle stream
512
The username of the owner of the bundle stream for personal
513
streams or an empty string for public and team streams.
515
The name of the team that owsn the bundle stream for team
516
streams or an empty string for public and personal streams.
518
Number of bundles that are in this stream
524
Rules for bundle stream access
525
------------------------------
526
The following rules govern bundle stream download access rights:
527
- all anonymous streams are accessible
528
- personal streams are accessible to owners
529
- team streams are accessible to team members
531
if self.user and self.user.is_superuser:
532
bundle_streams = BundleStream.objects.all()
534
bundle_streams = BundleStream.objects.accessible_by_principal(
538
'pathname': bundle_stream.pathname,
539
'name': bundle_stream.name,
540
'user': bundle_stream.user.username if bundle_stream.user else "",
541
'group': bundle_stream.group.name if bundle_stream.group else "",
542
'bundle_count': bundle_stream.bundles.count(),
544
for bundle_stream in bundle_streams
547
def bundles(self, pathname):
551
`bundles` (`pathname`)
555
List all bundles in a specified bundle stream
560
The pathname of the bundle stream to query. This argument
561
*MUST* designate an existing stream or Fault(404, "...") is
562
raised. The user *MUST* have access to this stream or
563
Fault(403, "...") is raised.
567
This function returns an XML-RPC array of XML-RPC structs with
568
the following fields:
570
`uploaded_by`: string
571
The username of the user that uploaded this bundle or
572
empty string if this bundle was uploaded anonymously.
573
`uploaded_on`: datetime
574
The timestamp when the bundle was uploaded
575
`content_filename`: string
576
The filename of the original bundle file
577
`content_sha1`: string
578
The SHA1 hash if the content of the bundle
580
The size of the content
581
`is_deserialized`: bool
582
True if the bundle was de-serialized successfully, false otherwise
583
`associated_job`: int
584
The job with which this bundle is associated
592
- Bundle stream not found
593
- Listing bundles in this bundle stream is not permitted
595
Rules for bundle stream access
596
------------------------------
597
The following rules govern bundle stream download access rights:
598
- all anonymous streams are accessible
599
- personal streams are accessible to owners
600
- team streams are accessible to team members
604
if self.user and self.user.is_superuser:
605
bundle_stream = BundleStream.objects.get(pathname=pathname)
607
bundle_stream = BundleStream.objects.accessible_by_principal(self.user).get(pathname=pathname)
608
for bundle in bundle_stream.bundles.all().order_by("uploaded_on"):
611
job = TestJob.objects.get(_results_bundle=bundle)
613
except TestJob.DoesNotExist:
616
'uploaded_by': bundle.uploaded_by.username if bundle.uploaded_by else "",
617
'uploaded_on': bundle.uploaded_on,
618
'content_filename': bundle.content_filename,
619
'content_sha1': bundle.content_sha1,
620
'content_size': bundle.content.size,
621
'is_deserialized': bundle.is_deserialized,
622
'associated_job': job_id
624
except BundleStream.DoesNotExist:
625
raise xmlrpclib.Fault(errors.NOT_FOUND, "Bundle stream not found")
628
@xml_rpc_signature('str')
629
def get_test_names(self, device_type=None):
633
`get_test_names` ([`device_type`]])
637
Get the name of all the tests that have run on a particular device type.
641
`device_type`: string
642
The type of device the retrieved test names should apply to.
646
This function returns an XML-RPC array of test names.
650
for test in Test.objects.filter(
651
test_runs__attributes__name='target.device_type',
652
test_runs__attributes__value=device_type).distinct():
653
test_names.append(test.test_id)
655
for test in Test.objects.all():
656
test_names.append(test.test_id)
659
def deserialize(self, content_sha1):
663
`deserialize` (`content_sha1`)
667
Deserialize bundle on the server
671
`content_sha1`: string
672
SHA1 hash of the content of the bundle to download. This
673
*MUST* designate an bundle or ``Fault(404, "...")`` is raised.
677
True - deserialization okay
678
False - deserialization not needed
688
bundle = Bundle.objects.get(content_sha1=content_sha1)
689
except Bundle.DoesNotExist:
690
raise xmlrpclib.Fault(errors.NOT_FOUND, "Bundle not found")
691
if bundle.is_deserialized:
694
if bundle.is_deserialized is False:
695
raise xmlrpclib.Fault(
697
bundle.deserialization_error.error_message)
700
def make_stream(self, pathname, name):
704
`make_stream` (`pathname`, `name`)
708
Create a bundle stream with the specified pathname
713
The pathname must refer to an anonymous stream
715
The name of the stream (free form description text)
724
Pathname does not designate an anonymous stream
726
Bundle stream with the specified pathname already exists
732
# Work around bug https://bugs.launchpad.net/lava-dashboard/+bug/771182
733
# Older clients would send None as the name and this would trigger an
734
# IntegrityError to be raised by BundleStream.objects.create() below
735
# which in turn would be captured by the fault handler and reported as
736
# an unrelated issue to the user. Let's work around that by using an
737
# empty string instead.
741
user_name, group_name, slug, is_public, is_anonymous = BundleStream.parse_pathname(pathname)
742
except ValueError as ex:
743
raise xmlrpclib.Fault(errors.FORBIDDEN, str(ex))
745
# Start with those to simplify the logic below
748
if is_anonymous is False:
749
if self.user is not None:
750
assert is_anonymous is False
751
assert self.user is not None
752
if user_name is not None:
753
if not self.user.is_superuser:
754
if user_name != self.user.username:
755
raise xmlrpclib.Fault(
757
"Only user {user!r} could create this stream".format(user=user_name))
758
user = self.user # map to real user object
759
elif group_name is not None:
761
if self.user.is_superuser:
762
group = Group.objects.get(name=group_name)
764
group = self.user.groups.get(name=group_name)
765
except Group.DoesNotExist:
766
raise xmlrpclib.Fault(
768
"Only a member of group {group!r} could create this stream".format(group=group_name))
770
assert is_anonymous is False
771
assert self.user is None
772
raise xmlrpclib.Fault(
773
errors.FORBIDDEN, "Only anonymous streams can be constructed (you are not signed in)")
775
assert is_anonymous is True
776
assert user_name is None
777
assert group_name is None
778
if self.user is not None:
781
# Hacky but will suffice for now
782
user = User.objects.get_or_create(username="anonymous-owner")[0]
784
bundle_stream = BundleStream.objects.create(
789
is_anonymous=is_anonymous,
791
except IntegrityError:
792
raise xmlrpclib.Fault(
794
"Stream with the specified pathname already exists")
796
return bundle_stream.pathname
798
def _get_filter_data(self, filter_name):
799
match = re.match("~([-_A-Za-z0-9]+)/([-_A-Za-z0-9]+)", filter_name)
801
raise xmlrpclib.Fault(errors.BAD_REQUEST, "filter_name must be of form ~owner/filter-name")
802
owner_name, filter_name = match.groups()
804
owner = User.objects.get(username=owner_name)
805
except User.NotFound:
806
raise xmlrpclib.Fault(errors.NOT_FOUND, "user %s not found" % owner_name)
808
filter = TestRunFilter.objects.get(owner=owner, name=filter_name)
809
except TestRunFilter.NotFound:
810
raise xmlrpclib.Fault(errors.NOT_FOUND, "filter %s not found" % filter_name)
811
if not filter.public and self.user != owner:
813
raise xmlrpclib.Fault(
814
errors.FORBIDDEN, "forbidden")
816
raise xmlrpclib.Fault(
817
errors.AUTH_REQUIRED, "authentication required")
818
return filter.as_data()
820
def get_filter_results(self, filter_name, count=10, offset=0):
826
get_filter_results(filter_name, count=10, offset=0)
831
Return information about the test runs and results that a given filter
838
The name of a filter in the format ~owner/name.
840
The maximum number of matches to return.
842
Skip over this many results.
847
A list of "filter matches". A filter match describes the results of
848
matching a filter against one or more test runs::
851
'tag': either a stringified date (bundle__uploaded_on) or a build number
854
'link': link-to-test-run,
855
'passes': int, 'fails': int, 'skips': int, 'total': int,
856
# only present if filter specifies cases for this test:
857
'specific_results': [{
858
'test_case_id': test_case_id,
859
'link': link-to-test-result,
860
'result': pass/fail/skip/unknown,
861
'measurement': string-containing-decimal-or-None,
865
# Only present if filter does not specify tests:
871
filter_data = self._get_filter_data(filter_name)
872
matches = evaluate_filter(self.user, filter_data, descending=False)
873
matches = matches[offset:offset + count]
874
return [match.serializable() for match in matches]
876
def get_filter_results_since(self, filter_name, since=None):
882
get_filter_results_since(filter_name, since=None)
887
Return information about the test runs and results that a given filter
888
matches that are more recent than a previous match -- in more detail,
889
results where the ``tag`` is greater than the value passed in
892
The idea of this method is that it will be called from a cron job to
893
update previously accessed results. Something like this::
895
previous_results = json.load(open('results.json'))
896
results = previous_results + server.dashboard.get_filter_results_since(
897
filter_name, previous_results[-1]['tag'])
898
... do things with results ...
899
json.save(results, open('results.json', 'w'))
901
If called without passing ``since`` (or with ``since`` set to
902
``None``), this method returns up to 100 matches from the filter. In
903
fact, the matches are always capped at 100 -- so set your cronjob to
904
execute frequently enough that there are less than 100 matches
905
generated between calls!
911
The name of a filter in the format ~owner/name.
913
The 'tag' of the most recent result that was retrieved from this
919
A list of "filter matches". A filter match describes the results of
920
matching a filter against one or more test runs::
923
'tag': either a stringified date (bundle__uploaded_on) or a build number
926
'link': link-to-test-run,
927
'passes': int, 'fails': int, 'skips': int, 'total': int,
928
# only present if filter specifies cases for this test:
929
'specific_results': [{
930
'test_case_id': test_case_id,
931
'link': link-to-test-result,
932
'result': pass/fail/skip/unknown,
933
'measurement': string-containing-decimal-or-None,
937
# Only present if filter does not specify tests:
943
filter_data = self._get_filter_data(filter_name)
944
matches = evaluate_filter(self.user, filter_data, descending=False)
945
if since is not None:
946
if filter_data.get('build_number_attribute') is not None:
948
since = datetime.datetime.strptime(since, "%Y-%m-%d %H:%M:%S.%f")
950
raise xmlrpclib.Fault(
951
errors.BAD_REQUEST, "cannot parse since argument as datetime")
952
matches = matches.since(since)
953
matches = matches[:100]
954
return [match.serializable() for match in matches]
956
# Mapper used by the legacy URL
957
legacy_mapper = Mapper()
958
legacy_mapper.register_introspection_methods()
959
legacy_mapper.register(DashboardAPI, '')
added
removed
dashboard_app/xmlrpc.py
