~dooferlad/launchpad-work-items-tracker/bug1092862

#!/usr/bin/python
# -*- coding: UTF-8 -*-
# Copyright (C) 2012 Linaro Ltd.
# Author: Loïc Minier <loic.minier@linaro.org>
# License: GPL-3

import jira

from bs4 import BeautifulSoup
import logging
import optparse
import os
import re
import simplejson
import sys
import urllib2

logger = logging.getLogger("linaroroadmap")

def dbg(msg):
    '''Print out debugging message if debugging is enabled.'''
    logger.debug(msg)

class InMemCollector:
    def __init__(self):
        self.lanes = []
        self.cards = []

    def store_lane(self, lane):
        self.lanes.append(lane)

    def store_card(self, card):
        self.cards.append(card)

    def lane_is_collected(self, lane_id):
        for l in self.lanes:
            if l.lane_id == lane_id:
                return True
        return False

def kanban_request(opts, relpathname, method='GET', **kwargs):
    request = urllib2.Request(
        '%s/%s.json?_m=%s' % (opts.kanban_api_url, relpathname, method))
    if opts.kanban_token:
        request.add_header('X-KanbanToolToken', opts.kanban_token)
    request_data = None
    if kwargs.keys():
        request.add_header('Content-Type', 'application/json')
        request_data = simplejson.dumps(kwargs)
        print request_data
    response_data = urllib2.urlopen(request, request_data)
    return simplejson.load(response_data)

def get_papyrs_page(papyrs_url, token):
    url = '%s?json&auth_token=%s' % (papyrs_url, token)
    return simplejson.load(urllib2.urlopen(url))

def get_kanban_boards(opts):
    return kanban_request(opts, 'boards')

def get_kanban_board(opts, board_id):
    return kanban_request(opts, 'boards/%s' % board_id)

def get_kanban_tasks(opts, board_id):
    return kanban_request(opts, 'boards/%s/tasks' % board_id)

def get_kanban_task(opts, board_id, task_id):
    return kanban_request(opts, 'boards/%s/tasks/%s' % (board_id, task_id))

def put_kanban_task(opts, board_id, task_id, **kwargs):
    return kanban_request(opts, 'boards/%s/tasks/%s' % (board_id, task_id), method='PUT', **kwargs)

def main():
    # TODO: add support for passing a card id or papyrs URL
    parser = optparse.OptionParser(usage="%prog")
    parser.add_option("--kanban-api-url", dest="kanban_api_url",
        default="https://linaro.kanbantool.com/api/v1")
    parser.add_option("--jira-api-url", dest="jira_api_url",
        default="http://cards.linaro.org/rest/api/2")
    # defaults are read-only ~linaro-infrastructure tokens
    parser.add_option("--kanban-token", dest="kanban_token",
        default="9F209W7Y84TE")
    parser.add_option("--papyrs-token", dest="papyrs_token",
        default="868e9088b53c")
    parser.add_option("--jira-username", dest="jira_username",
        default="robot")
    parser.add_option("--jira-password", dest="jira_password",
        default="cuf4moh2")
    parser.add_option("--jira-project", dest="jira_project_name",
        default="CARD")
    parser.add_option("--jira-issuetype", dest="jira_issuetype_name",
        default="Roadmap Card")
    parser.add_option("--board-id", dest="board_id", default="10721")
    parser.add_option('--debug', action='store_true', default=True,
        help='Enable debugging output in parsing routines')
    parser.add_option('--board',
        help='Board id at Kanban Tool', dest='board', default='10721')
    opts, args = parser.parse_args()

    if os.environ.get("DEBUG", None) is not None:
        opts.debug = True

    if len(args) != 0:
        parser.error("You can not pass any argument")

    if opts.kanban_token is None:
        sys.stderr.write("No Kanbantool API token given")
    if opts.papyrs_token is None:
        sys.stderr.write("No Papyrs API token given")

    # logging setup
    logger = logging.getLogger()
    ch = logging.StreamHandler()
    formatter = logging.Formatter("%(asctime)s %(message)s")
    ch.setFormatter(formatter)
    logger.addHandler(ch)
    if opts.debug:
        logger.setLevel(logging.DEBUG)

    boards = get_kanban_boards(opts)
    # dump
    for board in boards:
        board = board['board']
        dbg('Found board "%s" with id %s' % (board['name'], board['id']))
    dbg('')

    assert 1 == len(filter(lambda b: str(b['board']['id']) == opts.board_id, boards)), \
        'Expected exactly one board with id %s' % opts.board_id

    board = get_kanban_board(opts, opts.board_id)
    board = board['board']

    workflow_stages = board['workflow_stages']
    # ideally order wouldn't matter but the "position" field of our workflow stages
    # is bogus (always 1) so we can't use it
    leaf_workflow_stages = []
    for workflow_stage in workflow_stages:
        childs = filter(
            lambda ws: ws['parent_id'] == workflow_stage['id'], workflow_stages)
        if not childs:
            # build a name list for leaf workflow stages
            name = []
            id = workflow_stage['id']
            while True:
                ws = filter(lambda ws: ws['id'] == id, workflow_stages)[0]
                if ws['name'] is None:
                    break
                name = [ws['name']] + name
                id = ws['parent_id']
            leaf_workflow_stages.append((workflow_stage['id'], name))
    # dump
    for id, name in leaf_workflow_stages:
        pretty_name = "/".join(name)
        dbg('Found leaf workflow stage %s with id %s' % (pretty_name, id))
    dbg('')

    card_types = board['card_types']
    # dump
    for card_type in card_types:
        dbg('Found card type %s with id %s' % (card_type['name'], card_type['id']))
    dbg('')

    def get_leaf_workflow_stage_name(worfklow_stage_id):
        return [name
                    for id, name
                    in leaf_workflow_stages
                    if id == worfklow_stage_id][0]

    def get_card_type_name(card_type_id):
        return [card_type['name']
                    for card_type
                    in card_types
                    if card_type['id'] == card_type_id][0]

    def filter_tasks(task):
        # ignore tasks in Legend and Deferred workflow stages
        lwsn = get_leaf_workflow_stage_name(task['workflow_stage_id'])
        if lwsn in (['Legend'], ['Deferred']):
            dbg('Ignoring task %s in workflow stage %s'
                    % (task['external_id'], "/".join(lwsn)))
            return False
        # ignore tasks with Summit and Unknown card type names
        card_type_name = get_card_type_name(task['card_type_id'])
        if card_type_name in ('Summits', 'Unknown'):
            dbg('Ignoring task %s with card type name %s'
                    % (task['external_id'], card_type_name))
            return False
        return True

    tasks = get_kanban_tasks(opts, opts.board_id)
    tasks = [t['task'] for t in tasks if filter_tasks(t['task'])]
    # dump
    for task in tasks:
        dbg('Found task %s with id %s, workflow_stage_id %s, priority %s, '
            'card_type_id %s, custom_field_2 %s, and external_id %s'
                % (task['name'], task['id'], task['workflow_stage_id'],
                   task['priority'], task['card_type_id'],
                   task['custom_field_2'], task['external_id']))

    CARD_TYPE_NAMES_TO_PREFIXES = {
        'LAVA': 'LAVA',
        'Android': 'ANDROID',
        'Linux & Ubuntu': 'LINUX',
        'TCWG': 'TCWG',
        'GWG': 'GWG',
        'MMWG': 'MMWG',
        'KWG': 'KWG',
        'PMWG': 'PMWG',
        'OCTO': 'OCTO',
    }

    # check consistency of external_id with external_link and custom_field_2
    # (papyrs URL), and of external_id with card_type name
    for task in tasks:
        external_id = task['external_id']
        papyrs_url = task['custom_field_2']
        external_link = task['external_link']
        assert papyrs_url == 'https://linaro.papyrs.com/%s' % external_id, \
            'Incorrect papyrs URL %s for task %s' % (papyrs_url, external_id)
        assert external_link == 'http://status.linaro.org/card/%s' % external_id, \
            'Incorrect external_link %s for task %s' % (external_link, external_id)
        card_type_name = get_card_type_name(task['card_type_id'])
        prefix = CARD_TYPE_NAMES_TO_PREFIXES[card_type_name]
        assert external_id.startswith(prefix), \
            'Incorrect card type prefix %s for task %s' % (prefix, external_id)

    # verify papyrs pages
    #for task in tasks:
    for task in []:
        external_id = task['external_id']
        papyrs_url = task['custom_field_2']
        dbg('Fetching card %s' % task['name'])
        papyrs_json = get_papyrs_page(papyrs_url, opts.papyrs_token)

        try:
            # number of columns
            ncols = len(papyrs_json)
            assert ncols == 2, 'Expected exactly two columns but got %s' % len(ncols)

            # first column
            col0 = papyrs_json[0]
            p0 = col0[0]
            classname = p0['classname']
            assert classname == 'Heading', \
                "First paragraph of first column should be a a heading but is %s" % classname
            assert p0['text'] == p0['html'], \
                "Expected text (%s) and HTML (%s) to be identical for first heading" % (p0['text'], p0['html'])
            assert p0['text'] == task['name'], \
                'Mismatch between first heading (%s) and task (%s)' % (p0['text'], task['name'])
            for p in col0[1:-2]:
                assert p['classname'] in ('Heading', 'Paragraph'), \
                    'Got unexpected classname %s' % p['classname']
                if p['classname'] == 'Heading':
                    assert p['text'] == p['html'], \
                        'Expected heading HTML (%s) to match text (%s)' % (p['html'], p['text'])
                if p['classname'] == 'Paragraph':
                    soup = BeautifulSoup('<root>%s</root>' % p['html'], 'xml')
                    for tag in soup.root.find_all(True):
                        assert tag.name in ('font', 'b', 'a', 'ul', 'ol', 'li', 'br', 'p', 'span', 'div', 'u'), 'Unexpected tag %s' % tag.name

            # second column
            pm1 = col0[-1]
            assert pm1['classname'] == 'Discuss', \
                'Expect last classname to be Discuss but got %s' % pm1['classname']

            col1 = papyrs_json[1]
            skip_next_paragraph = False
            nattachs = 0
            for p in col1:
                if p['classname'] in ('Checklist', 'Twitters', 'Navigation'):
                    pass
                elif p['classname'] == 'Attachment':
                    nattachs += 1
                elif p['classname'] == 'Heading' and p['text'] == 'Attachments':
                    pass
                elif p['classname'] == 'Heading' and p['text'] == 'Metadata':
                    skip_next_paragraph = True
                elif p['classname'] == 'Paragraph' and skip_next_paragraph:
                    skip_next_paragraph = False
                else:
                    assert False, 'Unexpected paragraph %s' % p
            if nattachs > 0:
                dbg('Found %s attachment(s) on card %s' % (nattachs, task['name']))
        except Exception, e:
            dbg(e)

    # query jira data
    jira_project_result = jira.do_request(opts, 'project/%s' % opts.jira_project_name)
    jira_statuses_result = jira.do_request(opts, 'status')
    jira_fields_result = jira.do_request(opts, 'field')
    # not allowed
    #jira_securitylevels_result = jira.do_request(opts, 'securitylevel')
    jira_priorities_result = jira.do_request(opts, 'priority')

    def search_jira_id(jira_result, name):
        return [r['id'] for r in jira_result if r['name'] == name][0]

    # http://cards.linaro.org/rest/api/2/project/CARD has id 10000
    #jira_project_id = 10000
    jira_project_id = jira_project_result['id']
    # issuetype for "Roadmap Card" http://cards.linaro.org/rest/api/2/issuetype/9
    #jira_issuetype_id = 9
    jira_issuetype_id = search_jira_id(jira_project_result['issueTypes'], opts.jira_issuetype_name)
    dbg('Found id %s for %s issueType' % (jira_issuetype_id, opts.jira_issuetype_name))
    for component in jira_project_result['components']:
        dbg('Found component %s with id %s' % (component['name'], component['id']))
    for version in jira_project_result['versions']:
        dbg('Found version %s with id %s' % (version['name'], version['id']))
    for status in jira_statuses_result:
        dbg('Found status %s with id %s' % (status['name'], status['id']))

    TYPE_TO_COMPONENT = {
        'LAVA': 'LAVA',
        'Android': 'Android',
        'Linux & Ubuntu': 'Linux & Ubuntu',
        'TCWG': 'Toolchain WG',
        'GWG': 'Graphics WG',
        'MMWG': 'Multimedia WG',
        'KWG': 'Kernel WG',
        'PMWG': 'Power Management WG',
        'OCTO': 'OCTO',
    }

    STAGE_TO_STATUS = {
        'New/Draft': 'New/Drafting',
        'New/Needs Work': 'New/Drafting',
        'New/TSC Reviewed': 'New/Reviewed',
        '2012Q1/Done': 'Approved',
        '2012Q1/Ready': 'Approved',
        '2012Q2/Forecast': 'Approved',
        '2012Q3/Forecast': 'Approved',
        '2012H2/Forecast': 'Approved',
        '2013/Forecast': 'Approved',
    }

    STAGE_TO_VERSION = {
        'New/Draft': None,
        'New/Needs Work': None,
        'New/TSC Reviewed': None,
        '2012Q1/Done': '2012Q1',
        '2012Q1/Ready': '2012Q1',
        '2012Q2/Forecast': '2012Q2',
        '2012Q3/Forecast': '2012Q3',
        '2012H2/Forecast': '2012H2',
        '2013/Forecast': '2013',
    }

    PRIORITY_MAP = {
        -1: 'Minor',
        0: 'Major',
        1: 'Critical',
    }

    # actual copy
    for task in tasks:
        print task['name']
        print task['external_id']
        external_id = task['external_id']
        papyrs_url = task['custom_field_2']
        papyrs_json = get_papyrs_page(papyrs_url, opts.papyrs_token)
        # first column
        col0 = papyrs_json[0]
        p0 = col0[0]
        # assemble HTML of description
        html = ""
        for p in col0[1:-1]:
            if p['classname'] == 'Heading':
                html += '<h1>%s</h1>\n' % p['text']
            if p['classname'] == 'Paragraph':
                html += '%s\n' % p['html']
        html = '{html}\n%s{html}\n' % html

        stage = "/".join(get_leaf_workflow_stage_name(task['workflow_stage_id']))
        status = STAGE_TO_STATUS[stage]
        version = STAGE_TO_VERSION[stage]
        type_name = get_card_type_name(task['card_type_id'])
        component = TYPE_TO_COMPONENT[type_name]
        priority = PRIORITY_MAP[task['priority']]

        fields = {'project': {'id': jira_project_id},
                  'summary': task['name'],
                  'issuetype': {'id': jira_issuetype_id},
                  'description': html,
                  'components': [{'id': search_jira_id(jira_project_result['components'], component)}],
                  search_jira_id(jira_fields_result, 'Alias Card ID'): task['external_id'],
                  # XXX hardcoded default security level; also, can't set security level to Public via API
                  #'security': {'id': search_jira_id(jira_securitylevels_result, 'Public')},
                  'priority': {'id': search_jira_id(jira_priorities_result, priority)},
                 }
                  #'status': search_jira_id(jira_statuses_result, status),
        if version:
            fields['fixVersions'] = [{'id': search_jira_id(jira_project_result['versions'], version)}]
        dbg('Uploading card %s' % fields)
        print jira.do_request(opts, 'issue', fields=fields)

if __name__ == "__main__":
    main()