~sinzui/juju-ci-tools/cloudsigma-lib

#!/usr/bin/env python

from __future__ import print_function

from argparse import ArgumentParser

from ConfigParser import ConfigParser

import json

import os

import sys

from time import sleep

import urlparse

from boto.s3.connection import S3Connection

import requests

from requests.auth import HTTPBasicAuth

from jujuci import(

    get_build_data,

    add_credential_args,

    get_credentials,

    get_job_data,

    JENKINS_URL,

)

from utility import until_timeout

__metaclass__ = type

class JenkinsBuild:

    """

    Retrieves Jenkins build information

    """

    def __init__(self, credentials, job_name, jenkins_url, build_info):

        """

        :param credentials: Jenkins credentials

        :param job_name:  Jenkins job name

        :param jenkins_url: Jenkins server URL

        :param build_info: Jenkins build info

        :return: None

        """

        self.credentials = credentials

        self.jenkins_url = jenkins_url

        self.job_name = job_name

        self.build_info = build_info

    @classmethod

    def factory(cls, credentials, job_name, build_number=None, url=None):

        """

        :param credentials: Jenkins credentials

        :param job_name: Jenkins job name

        :param build_number: Jenkins build number

        :param url: Jenkins url

        :return:

        """

        url = url or JENKINS_URL

        build_info = (get_build_data(url, credentials, job_name, build_number)

                      if build_number else None)

        return cls(credentials, job_name, url, build_info)

    def get_build_info(self, build_number=None):

        """

        Gets build info from the Jenkins server

        :rtype: dict

        """

        build_number = build_number or self.get_build_number()

        self.build_info = get_build_data(

            self.jenkins_url, self.credentials, self.job_name, build_number)

        return self.build_info

    def is_build_completed(self):

        """Check if the build is completed and return boolean."""

        build_info = self.get_build_info()

        return not build_info['building']

    @property

    def result(self):

        """

        Returns the test result string

        :return: SUCCESS, FAILURE, ABORTED, NOT_BUILT, SUCCESS, UNSTABLE ...

        :rtype: str

        """

        return self.build_info.get('result')

    def get_console_text(self):

        """

        Return Jenkins build console log

        :rtype: str

        """

        log_url = urlparse.urljoin(self.build_info['url'], 'consoleText')

        return requests.get(

            log_url, auth=HTTPBasicAuth(

                self.credentials.user, self.credentials.password)).text

    def get_last_completed_build_number(self):

        """

        Returns latest Jenkins build number

        :rtype: int

        """

        job_info = get_job_data(

            self.jenkins_url, self.credentials, self.job_name)

        return job_info['lastCompletedBuild']['number']

    def artifacts(self):

        """

        Returns the filename and the content of artifacts

        :return: filename and artifacts content

        :rtype: tuple

        """

        relative_paths = [(x['relativePath'], x['fileName']) for x in

                          self.build_info['artifacts']]

        auth = HTTPBasicAuth(self.credentials.user, self.credentials.password)

        for path, filename in relative_paths:

            url = self._get_artifact_url(path)

            content = requests.get(url, auth=auth).content

            yield filename, content

    def _get_artifact_url(self, relative_path):

        """

        :return: List of artifact URLs

        :rtype: list

        """

        return self.build_info['url'] + 'artifact/' + relative_path

    def get_build_number(self):

        return self.build_info.get('number')

    def set_build_number(self, build_number):

        self.get_build_info(build_number)

class S3:

    """

    Used to store an object in S3

    """

    def __init__(self, directory, access_key, secret_key, conn, bucket):

        self.dir = directory

        self.access_key = access_key

        self.secret_key = secret_key

        self.conn = conn

        self.bucket = bucket

    @classmethod

    def factory(cls, bucket, directory):

        access_key, secret_key = get_s3_access()

        conn = S3Connection(access_key, secret_key)

        bucket = conn.get_bucket(bucket)

        return cls(directory, access_key, secret_key, conn, bucket)

    def store(self, filename, data, headers=None):

        """

        Stores an object in S3.

        :param filename: filename of the object

        :param data: The content to be stored in S3

        :rtype: bool

        """

        if not data:

            return False

        path = os.path.join(self.dir, filename)

        key = self.bucket.new_key(path)

        # This will store the data.

        key.set_contents_from_string(data, headers=headers)

        return True

class S3Uploader:

    """

    Uploads the result of a Jenkins job to S3.

    """

    def __init__(self, s3, jenkins_build):

        self.s3 = s3

        self.jenkins_build = jenkins_build

    @classmethod

    def factory(cls, credentials, jenkins_job, build_number, bucket,

                directory):

        """

        Creates S3Uploader.

        :param credentials: Jenkins credential

        :param jenkins_job: Jenkins job name

        :param build_number: Jenkins build number

        :param bucket: S3 bucket name

        :param directory: S3 directory name

        :rtype: S3Uploader

        """

        s3 = S3.factory(bucket, directory)

        build_number = int(build_number) if build_number else build_number

        jenkins_build = JenkinsBuild.factory(

            credentials=credentials, job_name=jenkins_job,

            build_number=build_number)

        return cls(s3, jenkins_build)

    def upload(self):

        """Uploads Jenkins job results, console logs and artifacts to S3.

        :return: None

        """

        self.upload_test_results()

        self.upload_console_log()

        self.upload_artifacts()

    def upload_by_build_number(self, build_number=None, pause_time=120,

                               timeout=600):

        """

        Upload build_number's test result.

        :param build_number:

        :param pause_time: Pause time in seconds between polling.

        :param timeout: Timeout in seconds.

        :return: None

        """

        build_number = build_number or os.getenv('BUILD_NUMBER')

        if not build_number:

            raise ValueError('Build number is not set')

        self.jenkins_build.set_build_number(build_number)

        for _ in until_timeout(timeout):

            if self.jenkins_build.is_build_completed():

                break

            sleep(pause_time)

        else:

            raise Exception("Build fails to complete: {}".format(build_number))

        self.upload()

    def upload_all_test_results(self):

        """

        Uploads all the test results to S3. It starts with the build_number 1

        :return: None

        """

        latest_build_num = self.jenkins_build.get_last_completed_build_number()

        for build_number in range(1, latest_build_num + 1):

            self.jenkins_build.set_build_number(build_number)

            self.upload()

    def upload_last_completed_test_result(self):

        """Upload the latest test result to S3."""

        latest_build_num = self.jenkins_build.get_last_completed_build_number()

        self.jenkins_build.set_build_number(latest_build_num)

        self.upload()

    def upload_test_results(self):

        filename = self._create_filename('result-results.json')

        headers = {"Content-Type": "application/json"}

        self.s3.store(filename, json.dumps(

            self.jenkins_build.get_build_info(), indent=4), headers=headers)

    def upload_console_log(self):

        filename = self._create_filename('console-consoleText.txt')

        headers = {"Content-Type": "text/plain; charset=utf8"}

        self.s3.store(

            filename, self.jenkins_build.get_console_text(), headers=headers)

    def upload_artifacts(self):

        headers = {"Content-Type": "application/octet-stream"}

        for filename, content in self.jenkins_build.artifacts():

            filename = self._create_filename('log-' + filename)

            self.s3.store(filename, content, headers=headers)

    def _create_filename(self, filename):

        """

        Creates filename based on the combination of the job ID and the

        filename

        :return: Filename

        :rtype: str

        """

        return str(self.jenkins_build.get_build_number()) + '-' + filename

def get_s3_access():

    """

    Return S3 access and secret keys

    """

    s3cfg_path = os.path.join(

        os.getenv('HOME'), 'cloud-city/juju-qa.s3cfg')

    config = ConfigParser()

    config.read(s3cfg_path)

    access_key = config.get('default', 'access_key')

    secret_key = config.get('default', 'secret_key')

    return access_key, secret_key

def get_args(argv=None):

    parser = ArgumentParser()

    parser.add_argument('jenkins_job', help="Jenkins job name.")

    parser.add_argument(

        'build_number', default=None,

        help="Build to upload, can be a number, 'all' or 'latest'.")

    parser.add_argument('s3_bucket', help="S3 bucket name to store files.")

    parser.add_argument(

        's3_directory',

        help="Directory under the bucket name to store files.")

    add_credential_args(parser)

    args = parser.parse_args(argv)

    args.all = False

    args.latest = False

    if args.build_number == 'all':

        args.all = True

        args.build_number = None

    if args.build_number == 'latest':

        args.latest = True

        args.build_number = None

    if args.build_number:

        args.build_number = int(args.build_number)

    return args

def main(argv=None):

    args = get_args(argv)

    cred = get_credentials(args)

    uploader = S3Uploader.factory(

        cred, args.jenkins_job, args.build_number, args.s3_bucket,

        args.s3_directory)

    if args.build_number:

        print('Uploading build number {:d}.'.format(args.build_number))

        uploader.upload()

    elif args.all:

        print('Uploading all test results.')

        uploader.upload_all_test_results()

    elif args.latest:

        print('Uploading the latest test result.')

        print('WARNING: latest can be a moving target.')

        uploader.upload_last_completed_test_result()

if __name__ == '__main__':

    sys.exit(main(sys.argv[1:]))