← Back to branch summary

~andrewjbeach/juju-ci-tools/make-local-patcher

« back to all changes in this revision

Viewing changes to upload_jenkins_job.py

  • Committer: John George
  • Date: 2016-02-05 22:13:41 UTC
  • mto: This revision was merged to the branch mainline in revision 1275.
  • Revision ID: john.george@canonical.com-20160205221341-3v2px3j23ik1zw5r
virtual maas templates and setup script.

Show diffs side-by-side

added added

removed removed

Lines of Context:
upload_jenkins_job.py
4
4
 
5
5
from argparse import ArgumentParser
6
6
import json
7
 
from mimetypes import MimeTypes
8
7
import os
9
8
import sys
10
9
from time import sleep
28
27
__metaclass__ = type
29
28
 
30
29
 
31
 
CONSOLE_TEXT = 'consoleText'
32
 
RESULT_RESULTS = 'result-results.json'
33
 
 
34
 
 
35
30
class JenkinsBuild:
36
31
    """
37
32
    Retrieves Jenkins build information
93
88
        Return Jenkins build console log
94
89
        :rtype: str
95
90
        """
96
 
        log_url = urlparse.urljoin(self.build_info['url'], CONSOLE_TEXT)
 
91
        log_url = urlparse.urljoin(self.build_info['url'], 'consoleText')
97
92
        return requests.get(
98
93
            log_url, auth=HTTPBasicAuth(
99
94
                self.credentials.user, self.credentials.password)).text
175
170
    Uploads the result of a Jenkins job to S3.
176
171
    """
177
172
 
178
 
    def __init__(self, s3, jenkins_build, unique_id=None, no_prefixes=False,
179
 
                 artifact_file_ext=None):
 
173
    def __init__(self, s3, jenkins_build, unique_id=None):
180
174
        self.s3 = s3
181
175
        self.jenkins_build = jenkins_build
182
176
        self.unique_id = unique_id
183
 
        self.no_prefixes = no_prefixes
184
 
        self.artifact_file_ext = artifact_file_ext
185
177
 
186
178
    @classmethod
187
179
    def factory(cls, credentials, jenkins_job, build_number, bucket,
188
 
                directory, unique_id=None, no_prefixes=False,
189
 
                artifact_file_ext=None):
 
180
                directory, unique_id=None):
190
181
        """
191
182
        Creates S3Uploader.
192
183
        :param credentials: Jenkins credential
194
185
        :param build_number: Jenkins build number
195
186
        :param bucket: S3 bucket name
196
187
        :param directory: S3 directory name
197
 
        :param artifact_file_ext: List of artifact file extentions. If set,
198
 
        only artifact with these ejections will be uploaded.
199
188
        :rtype: S3Uploader
200
189
        """
201
190
        s3 = S3.factory(bucket, directory)
203
192
        jenkins_build = JenkinsBuild.factory(
204
193
            credentials=credentials, job_name=jenkins_job,
205
194
            build_number=build_number)
206
 
        return cls(s3, jenkins_build,
207
 
                   unique_id=unique_id, no_prefixes=no_prefixes,
208
 
                   artifact_file_ext=artifact_file_ext)
 
195
        return cls(s3, jenkins_build, unique_id)
209
196
 
210
197
    def upload(self):
211
198
        """Uploads Jenkins job results, console logs and artifacts to S3.
255
242
        self.upload()
256
243
 
257
244
    def upload_test_results(self):
258
 
        filename = self._create_filename(RESULT_RESULTS)
 
245
        filename = self._create_filename('result-results.json')
259
246
        headers = {"Content-Type": "application/json"}
260
247
        build_info = self.jenkins_build.get_build_info()
261
248
        if self.unique_id:
266
253
            headers=headers)
267
254
 
268
255
    def upload_console_log(self):
269
 
        filename = self._create_filename(CONSOLE_TEXT)
 
256
        filename = self._create_filename('console-consoleText.txt')
270
257
        headers = {"Content-Type": "text/plain; charset=utf8"}
271
258
        self.s3.store(
272
259
            filename, self.jenkins_build.get_console_text(), headers=headers)
273
260
 
274
 
    @staticmethod
275
 
    def make_headers(filename):
276
 
        mime = MimeTypes()
277
 
        mime.add_type('text/plain', '.log')
278
 
        mime.add_type('text/x-yaml', '.yaml')
279
 
        content_type, encoding = mime.guess_type(filename)
 
261
    def upload_artifacts(self):
280
262
        headers = {"Content-Type": "application/octet-stream"}
281
 
        if content_type:
282
 
            headers['Content-Type'] = content_type
283
 
        if encoding:
284
 
            headers['Content-Encoding'] = encoding
285
 
        return headers
286
 
 
287
 
    def upload_artifacts(self):
288
263
        for filename, content in self.jenkins_build.artifacts():
289
 
            if self.artifact_file_ext:
290
 
                if os.path.splitext(filename)[1] not in self.artifact_file_ext:
291
 
                    continue
292
 
            filename = self._create_filename(filename)
293
 
            headers = self.make_headers(filename)
 
264
            filename = self._create_filename('log-' + filename)
294
265
            self.s3.store(filename, content, headers=headers)
295
266
 
296
267
    def _create_filename(self, filename):
300
271
        :return: Filename
301
272
        :rtype: str
302
273
        """
303
 
        if self.no_prefixes:
304
 
            return filename
305
 
        # Rules for dirs with files from several job builds.
306
 
        if filename == CONSOLE_TEXT:
307
 
            filename = 'console-consoleText.txt'
308
 
        elif filename != RESULT_RESULTS:
309
 
            filename = 'log-' + filename
310
274
        if self.unique_id:
311
275
            return "{}-{}".format(self.unique_id, filename)
312
276
        return str(self.jenkins_build.get_build_number()) + '-' + filename
335
299
        '--unique-id',
336
300
        help='Unique ID to be used to generate file names. If this is not '
337
301
             'set, the parent build number will be used as a unique ID.')
338
 
    parser.add_argument(
339
 
        '--no-prefixes', action='store_true', default=False,
340
 
        help='Do not add prefixes to file names; the s3_directory is unique.')
341
 
    parser.add_argument(
342
 
        '--artifact-file-ext', nargs='+',
343
 
        help='Artifacts include file extentions. If set, only files with '
344
 
             'these extentions will be uploaded.')
345
302
    add_credential_args(parser)
346
303
    args = parser.parse_args(argv)
347
304
    args.all = False
362
319
    cred = get_credentials(args)
363
320
    uploader = S3Uploader.factory(
364
321
        cred, args.jenkins_job, args.build_number, args.s3_bucket,
365
 
        args.s3_directory, unique_id=args.unique_id,
366
 
        no_prefixes=args.no_prefixes, artifact_file_ext=args.artifact_file_ext)
 
322
        args.s3_directory, unique_id=args.unique_id)
367
323
    if args.build_number:
368
324
        print('Uploading build number {:d}.'.format(args.build_number))
369
325
        uploader.upload()