~vila/uci-engine/1-run-worker-no-jenkins

#!/usr/bin/env python
# Ubuntu CI Engine
# Copyright 2014 Canonical Ltd.

# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License version 3, as
# published by the Free Software Foundation.

# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranties of
# MERCHANTABILITY, SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR
# PURPOSE.  See the GNU Affero General Public License for more details.

# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import base64
import json
import os
import random
import shutil
import string
import subprocess
import sys
import textwrap

# Import the local charm-helpers
sys.path.insert(0, os.path.join(os.environ['CHARM_DIR'], 'lib'))
from charmhelpers import core
from charmhelpers import fetch

JENKINS_HOME = '/var/lib/jenkins'
JENKINS_USER = 'jenkins'
JENKINS_GROUP = 'nogroup'
NUM_EXECUTORS_LINE = '  <numExecutors>{}</numExecutors>\n'


def juju_info(msg, level='INFO'):
    core.hookenv.log(msg, level)


def _relation_ids(relation_types=('db',)):
    # accept strings or iterators
    if isinstance(relation_types, basestring):
        reltypes = [relation_types, ]
    else:
        reltypes = relation_types
    relids = []
    for reltype in reltypes:
        relid_cmd_line = ['relation-ids', '--format=json', reltype]
        json_relids = subprocess.check_output(relid_cmd_line).strip()
        if json_relids:
            relids.extend(json.loads(json_relids))
    return relids


def _relation_get(key):
    return subprocess.check_output(['relation-get', key]).strip()


def _relation_set(keyvalues, relation_id=None):
    args = ['relation-set']
    if relation_id:
        args.extend(['-r', relation_id])
    args.extend(["{}={}".format(k, v or '') for k, v in keyvalues.items()])
    subprocess.check_call(args)


def _service_name(config):
    unit = os.environ['JUJU_UNIT_NAME'].split('/')[0]
    for x in (':', '-', '/', '"', "'"):
        unit = unit.replace(x, '_')
    return unit


def _service_dir(config):
    unit = os.environ['JUJU_UNIT_NAME'].split('/')[0]
    for x in (':', '-', '/', '"', "'"):
        unit = unit.replace(x, '_')
    return os.path.join(config['install_root'], unit)


def _relation_file(config):
    relation = os.environ['JUJU_RELATION_ID']
    return os.path.join(_service_dir(config), 'juju-relations', relation)


def add_dependencies(config):
    juju_info('Adding dependencies.')
    sources = config['sources'].split(';')
    for source in sources:
        fetch.add_source(source)
    fetch.apt_update()
    for package in config['packages'].split():
        fetch.apt_install(package)


def create_jobs(config):
    juju_info('Creating jenkins jobs from templates.')
    template_dir = os.path.join(core.hookenv.charm_dir(), 'templates', 'jobs')
    jenkins_home = JENKINS_HOME
    jenkins_user = JENKINS_USER
    jenkins_group = JENKINS_GROUP
    job_list = os.listdir(template_dir)
    for job in job_list:
        if not job.endswith('.xml'):
            juju_info('Found invalid job file: {}'.format(job))
            continue

        job_name = job[:-4]
        job_dir = os.path.join(jenkins_home, 'jobs', job_name)
        job_path = os.path.join(job_dir, 'config.xml')
        juju_info('Creating job: {}'.format(job_path))
        if not os.path.exists(job_dir):
            os.mkdir(job_dir)
        shutil.copy(os.path.join(template_dir, job), job_path)
        cmd = ['chown', '-R', '{}.{}'.format(jenkins_user, jenkins_group),
               job_dir]
        subprocess.check_call(cmd)


def create_user(config):
    juju_info('Creating lander user from template.')
    if not config['bot_password']:
        juju_info('Creating random password for bot.')
        password = ''.join(
            [random.choice(string.ascii_letters) for n in xrange(16)])
        core.host.write_file(os.path.join(JENKINS_HOME,
                                          '.lander-bot_password'),
                             password,
                             owner=JENKINS_HOME,
                             group=JENKINS_GROUP,
                             perms=0600)
    else:
        password = config['bot_password']
    salt = ''.join([random.choice(string.ascii_letters) for n in xrange(6)])
    p1 = subprocess.Popen(['echo', '-n', '{}{{{}}}'.format(password, salt)],
                          stdout=subprocess.PIPE)
    salted = subprocess.check_output(['shasum', '-a', '256'], stdin=p1.stdout)
    salted = salted.split()[0]
    user_dir = os.path.join(JENKINS_HOME, 'users', config['bot_username'])
    if not os.path.exists(user_dir):
        os.mkdir(user_dir)
    with open(os.path.join(core.hookenv.charm_dir(), 'templates', 'users',
                           'config.xml')) as template_file:
        user_template = template_file.read()
    template_map = {'__USERNAME__': config['bot_username'],
                    '__PASSWORD__': ':'.join([salt, salted]),
                    '__EMAIL__': config['bot_email']}
    for key in template_map:
        user_template = user_template.replace(key, template_map[key])
    user_config_file = os.path.join(user_dir, 'config.xml')
    juju_info('Creating lander user config: {}'. format(user_config_file))
    core.host.write_file(user_config_file, user_template)
    cmd = ['chown', '-R', '{}.{}'.format(JENKINS_USER, JENKINS_GROUP),
           user_dir]
    subprocess.check_call(cmd)


def setup_plugins(config):
    juju_info('Installing plugins.')
    plugin_list = config['plugins'].split()
    plugins_site = config['plugins-site']
    for plugin in plugin_list:
        if ':' in plugin:
            (name, version) = plugin.split(':')
            # A specific version is being requested
            url = '{}/download/plugins/{}/{}/{}.hpi'.format(
                plugins_site, name, version, name)
            plugin_file = '{}-{}.hpi'.format(name, version)
        else:
            url = '{}/latest/{}.hpi'.format(plugins_site, plugin)
            plugin_file = '{}.hpi'.format(name)
        juju_info('Installing {} from {}'.format(plugin_file, url))
        plugin_file = os.path.join(JENKINS_HOME, 'plugins', plugin_file)
        cmd = ['wget', '-O', plugin_file, url]
        subprocess.check_call(cmd)
        cmd = ['chown', '-R', '{}.{}'.format(JENKINS_USER, JENKINS_GROUP),
               plugin_file]
        subprocess.check_call(cmd)


def _install_from_tarball(config, upgrade):
    juju_info('grabbing service from tarball...')
    sdir = _service_dir(config)
    if not upgrade:
        if os.path.exists(sdir):
            juju_info('deleting pre-existing service directory: %s' % sdir)
            shutil.rmtree(sdir)
        os.mkdir(sdir)
    cmd = 'curl %s | tar -xzC %s' % (config['tarball'], sdir)
    subprocess.check_call(cmd, shell=True)


def _install_from_bzr(config, upgrade):
    juju_info('grabbing service from bzr...')

    sdir = _service_dir(config)
    rev = config.get('revno', '')

    if upgrade:
        args = ['bzr', 'pull']
        if rev:
            args.extend(['-r', rev])
        subprocess.check_call(args, cwd=sdir)
        return

    if os.path.exists(sdir):
        juju_info('deleting pre-existing service directory: %s' % sdir)
        shutil.rmtree(sdir)

    args = ['bzr', 'branch']
    if rev:
        args.extend(['-r', rev])
    args.append(config['branch'])
    args.append(sdir)
    subprocess.check_call(args)


def install(config, upgrade=False):
    if config.get('vcs') == 'tarball':
        _install_from_tarball(config, upgrade)
    else:
        _install_from_bzr(config, upgrade)

    user = config.get('uid', 'nobody')
    group = config.get('gid', 'nogroup')
    core.host.mkdir('/var/log/ci-tickets', user, group, 0755)

def upgrade_charm(config):
    install(config, True)
    restart_jenkins(config)
    restart_upstart(config)


def create_unit_config(config):
    unit_config = config.get('unit-config')
    if unit_config:
        with open(os.path.join(_service_dir(config), 'unit_config'), 'w') as f:
            f.write(base64.b64decode(unit_config))


def create_upstart(config):
    template = textwrap.dedent('''
    #--------------------------------------------------------------
    # This file is managed by the jenkins-lander Juju charm
    #--------------------------------------------------------------
    description "starts a jenkins-lander worker"
    start on (local-filesystems and net-device-up IFACE=eth0)
    stop on runlevel [!12345]

    # If the process quits unexpectedly trigger respawn it
    respawn
    # unless it fails 15 times within 5 seconds
    respawn limit 15 5

    setuid {uid}
    setgid {gid}
    chdir {sdir}

    exec {main}
    ''')
    params = {
        'main': config['main'],
        'sdir': _service_dir(config),
        'uid': config.get('uid', 'nobody'),
        'gid': config.get('gid', 'nogroup'),
    }
    with open('/etc/init/%s.conf' % _service_name(config), 'w') as f:
        f.write(template.format(**params))
        # need this so the file is ready for the subprocess call
        f.flush()
        os.fsync(f.fileno())


def update_jenkins_master(config):
    juju_info('Updating master jenkins.')
    jenkins_master_config = os.path.join(JENKINS_HOME, 'config.xml')
    num_executors = config.get('num_executors', None)
    if num_executors and os.path.exists(jenkins_master_config):
        with open(jenkins_master_config) as in_file:
            contents = in_file.readlines()
        with open(jenkins_master_config, 'w') as out_file:
            for line in contents:
                out_file.write(line)
                if line.startswith('<hudson>'):
                    out_file.write(NUM_EXECUTORS_LINE.format(num_executors))


def restart_jenkins(config):
    juju_info('Restarting jenkins.')
    core.host.service_stop('jenkins')
    core.host.service_start('jenkins')


def restart_upstart(config):
    try:
        subprocess.check_call(['/sbin/restart', _service_name(config)])
    except:
        # if it wasn't running, restart fails, so just try to start
        subprocess.check_call(['/sbin/start', _service_name(config)])


def config_changed(config):
    juju_info("Config: %s" % config)
    add_dependencies(config)
    create_jobs(config)
    create_user(config)
    setup_plugins(config)
    create_unit_config(config)
    create_upstart(config)
    update_jenkins_master(config)
    restart_jenkins(config)
    restart_upstart(config)


def lander_jenkins_relation_joined(config):
    juju_info("Config: %s" % config)
    hostname = subprocess.check_output(['unit-get', 'private-address']).strip()
    if ',' in hostname:
        # We can't have multiple hostnames, take the first one
        juju_info('Found too many private-addresses {}'.format(hostname))
        hostname = hostname.split(',')[0]
        juju_info('Taking first private-address: {}'.format(hostname))
    lander_config = {
        'lander_username': config['bot_username'],
        'lander_password': config['bot_password'],
        'lander_hostname': hostname,
        'lander_port': 8080,  # the jenkins charm hardcodes 8080
        'lander_master_job': 'lander_master',
        'lander_job_token': 'BUILD_ME',
    }
    # Add the lander config to the relation environment
    _relation_set(lander_config)
    # Also save the lander config locally
    with open(os.path.join(_service_dir(config), 'lander_config.json'),
              'w') as f:
        f.write(json.dumps(lander_config))
        f.write('\n')

    rfile = _relation_file(config)
    rdir = os.path.dirname(rfile)
    if not os.path.exists(rdir):
        os.mkdir(rdir)
    with open(rfile, 'w') as f:
        unit = os.environ['JUJU_REMOTE_UNIT'].split('/')[0]
        f.write('%s: %s' % (unit, _relation_get('private-address')))


def lander_jenkins_relation_changed(config):
    lander_jenkins_relation_joined(config)


def lander_jenkins_relation_broken(config):
    rfile = _relation_file(config)
    juju_info('removing relation file: %s' % rfile)
    if os.path.exists(rfile):
        os.unlink(rfile)


def jenkins_extension_relation_joined(config):
    juju_info("jenkins_extension_relation_joined")
    juju_info("Config: %s" % config)


def jenkins_extension_relation_changed(config):
    juju_info("jenkins_extension_relation_changed")
    juju_info("Config: %s" % config)


def jenkins_extension_relation_broken(config):
    juju_info("jenkins_extension_relation_broken")
    juju_info("Config: %s" % config)


def amqp_relation_joined(config):
    _relation_set({
        'username': config['amqp-user'],
        'vhost': config['amqp-vhost'],
    })


def amqp_relation_changed(config):
    with open(os.path.join(_service_dir(config), 'amqp_config.py'), 'w') as f:
        f.write('# DO NOT EDIT. Generated by lander-jenkins charm hook\n')
        f.write('AMQP_USER = "%s"\n' % config['amqp-user'])
        f.write('AMQP_VHOST = "%s"\n' % config['amqp-vhost'])
        f.write('AMQP_HOST = "%s"\n' % _relation_get('private-address'))
        f.write('AMQP_PASSWORD = "%s"\n' % _relation_get('password'))


def amqp_relation_broken(config):
    os.unlink(os.path.join(_service_dir(config), 'amqp_config.py'))


def main():
    hook = os.path.basename(sys.argv[0])
    juju_info("Running hook: %s" % hook)

    hook_py = hook.replace('-', '_')
    funcs = globals()
    if hook_py not in funcs:
        print("Unknown hook: %s" % hook)
        return 1

    config = core.hookenv.config()
    try:
        return funcs[hook_py](config)
    except subprocess.CalledProcessError as e:
        juju_info('Error running: %s: %s' % (e.cmd, e.output))
        return e.returncode


if __name__ == '__main__':
    exit(main())