← Back to branch summary

~larry-e-works/uci-engine/amqp-to-kombu

« back to all changes in this revision

Viewing changes to cupstream2distro/citrain/publisher

  • Committer: Francis Ginther
  • Date: 2014-06-10 20:42:46 UTC
  • mto: This revision was merged to the branch mainline in revision 571.
  • Revision ID: francis.ginther@canonical.com-20140610204246-b1bsrik7nlcolqy7
Import lp:cupstream2distro rev 605.

Show diffs side-by-side

added added

removed removed

Lines of Context:
cupstream2distro/citrain/publisher
 
1
#!/usr/bin/python
 
2
# -*- coding: utf-8 -*-
 
3
# Copyright (C) 2012-2014 Canonical
 
4
#
 
5
# Authors:
 
6
#  Didier Roche
 
7
#
 
8
# This program is free software; you can redistribute it and/or modify it under
 
9
# the terms of the GNU General Public License as published by the Free Software
 
10
# Foundation; version 3.
 
11
#
 
12
# This program is distributed in the hope that it will be useful, but WITHOUT
 
13
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 
14
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
 
15
# details.
 
16
#
 
17
# You should have received a copy of the GNU General Public License along with
 
18
# this program; if not, write to the Free Software Foundation, Inc.,b
 
19
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 
20
 
 
21
import argparse
 
22
import logging
 
23
import os
 
24
import sys
 
25
 
 
26
 
 
27
from cupstream2distro import branchhandling, launchpadmanager, packageinppamanager, packagemanager, silomanager, tools
 
28
from cupstream2distro.stacks import get_stack_packaging_change_status
 
29
from cupstream2distro.tools import generate_xml_artefacts, get_previous_distro_version_from_config, mark_project_as_published
 
30
from cupstream2distro.settings import PUBLISHER_ARTEFACTS_FILENAME, PACKAGE_LIST_RSYNC_FILENAME_FORMAT, SRU_PPA, SILO_BUILTCHECKED, SILO_PUBLISHED, PROPOSED_URL
 
31
from cupstream2distro.utils import ignored
 
32
 
 
33
 
 
34
if __name__ == '__main__':
 
35
 
 
36
    parser = argparse.ArgumentParser(description="Watch for published package in a ppa. Create a xml artefact if manual copy is needed.")
 
37
 
 
38
    parser.add_argument("publisher", help="Launchpad nickname of the person publishing that package.")
 
39
    parser.add_argument("--ackpackaging", action='store_true', help="Publish even if there are packaging change. Only done after manual reviews of the diff.")
 
40
    parser.add_argument("--ignoremissingprojects", action='store_true', help="Ignore if some projects that are going to be published are missing from the initial set (can be a second publication for instance).")
 
41
    parser.add_argument("--ignorestep", action='store_true', help="Ignore if previous step didn't successfully end.")
 
42
    parser.add_argument("--ignoreversiondestination", action='store_true', help="Ignore if the latest version in destination doesn't match when prepare was started.")
 
43
    parser.add_argument('-d', '--debug', action='store_true', default=False, help="Enable debug infos")
 
44
 
 
45
    args = parser.parse_args()
 
46
 
 
47
    logging.basicConfig(level=logging.DEBUG if args.debug else logging.INFO,
 
48
                        format="%(asctime)s %(levelname)s %(message)s")
 
49
    if args.debug:
 
50
        logging.debug("Debug mode enabled")
 
51
 
 
52
    silo_config = silomanager.load_config()
 
53
    silomanager.set_config_status(silo_config, "Publishing")
 
54
    series = launchpadmanager.get_resource_from_token(silo_config["global"]["series"]).name
 
55
    src_ppa = launchpadmanager.get_resource_from_token(silo_config["global"]["ppa"])
 
56
    src_ppa_name = "{}/{}".format(src_ppa.owner.name, src_ppa.name)
 
57
    src_pocket = 'Release'
 
58
    dest_link = silo_config["global"]["dest"]
 
59
    publisher_nickname = args.publisher
 
60
 
 
61
    base_path = os.path.abspath('.')
 
62
 
 
63
    if silomanager.get_config_step(silo_config) != SILO_BUILTCHECKED:
 
64
        if args.ignorestep:
 
65
            logging.info("Last step didn't finish successfully, but the ignore flag is set.")
 
66
        else:
 
67
            logging.error("Last step didn't finish successfully. You need to either ignore that the previous step didn't finished successfully or ensuring that prepare, build and checked passed.")
 
68
            silomanager.set_config_status(silo_config, "Can't publish: wrong status or parameters for job.")
 
69
            sys.exit(1)
 
70
        # reset it
 
71
        if not silomanager.set_config_step(silo_config, SILO_BUILTCHECKED):
 
72
            logging.error("Couldn't reset the step in the configuration file.")
 
73
            silomanager.set_config_status(silo_config, "Publication failed: couldn't reset status or parameters for job.")
 
74
            sys.exit(1)
 
75
    all_silo_projects = silomanager.get_all_projects(silo_config)
 
76
 
 
77
    # check the publisher exists
 
78
    try:
 
79
        launchpadmanager.get_person(publisher_nickname)
 
80
    except KeyError:
 
81
        message = "{} isn't a valid launchpad user name. Can't publish the silo".format(publisher_nickname)
 
82
        logging.error(message)
 
83
        silomanager.set_config_status(silo_config, "Publication failed: {}".format(message))
 
84
        sys.exit(1)
 
85
 
 
86
    logging.info("Publishing as {}.".format(publisher_nickname))
 
87
 
 
88
    # getting the archives and series objects
 
89
    is_sru = not launchpadmanager.is_series_current(series)
 
90
    dest_archive = launchpadmanager.get_resource_from_token(dest_link)
 
91
    if not launchpadmanager.is_dest_ubuntu_archive(dest_link):
 
92
        dest_pocket = 'Release'
 
93
        logging.info("Direct copy from {src_archive} ({src_pocket}) to {dest_archive} ({dest_pocket}) for {series} series".format(
 
94
                      src_archive=src_ppa.name, src_pocket=src_pocket, dest_archive=dest_archive.name, dest_pocket=dest_pocket,
 
95
                      series=series))
 
96
    else:
 
97
        logging.info("Copy from {src_archive} ({src_pocket}) to {series} series in distro".format(
 
98
                      src_archive=src_ppa.name, src_pocket=src_pocket, series=series))
 
99
        distro = launchpadmanager.get_resource_from_token(dest_link)
 
100
        distro_pocket = 'Proposed'
 
101
        src_ppa_pocket = src_pocket
 
102
        if is_sru:
 
103
            logging.info("This upload is part of a SRU. Please do not free the silo until the package lands in the archive!")
 
104
 
 
105
    manual_publish_cause_list = []
 
106
 
 
107
    all_packages_uploaded = packageinppamanager.get_all_packages_uploaded()
 
108
 
 
109
    # Generate global packaging change status
 
110
    manual_publish_cause_list.extend(get_stack_packaging_change_status(all_packages_uploaded))
 
111
 
 
112
    # we only go on manual publishing if we have something to publish
 
113
    if not args.ackpackaging and all_packages_uploaded and manual_publish_cause_list:
 
114
        logging.info("Don't upload the silo automatically.")
 
115
        generate_xml_artefacts("Publisher", manual_publish_cause_list, PUBLISHER_ARTEFACTS_FILENAME)
 
116
        silomanager.set_config_status(silo_config, "Publication needs action: some packaging changes needs manual acking. See them at {}".format(os.getenv('BUILD_URL')), add_url=False)
 
117
        sys.exit(0)
 
118
 
 
119
    packagelist_to_copy = []
 
120
    src_version_check_fail = False
 
121
    dest_version_check_fail = False
 
122
 
 
123
    error_pushing_branch = False
 
124
 
 
125
    published_packagelist = {}
 
126
 
 
127
    # check and generate metadata
 
128
    # TODO: we loop on all_packages_uploaded which is based on .project files. We should loop first on the config content (but still have a warning if we have more .project files)
 
129
    for (source, version, tip_rev, branch) in all_packages_uploaded:
 
130
        logging.info("Checking {} ({})".format(source, version))
 
131
        try:
 
132
            all_silo_projects.remove(source)
 
133
        except ValueError:
 
134
            # TODO: make that check first in its own loop
 
135
            # TODO2: add an option to say "it's ok, please remove from file system everything that's not mentionned in the config file"
 
136
            message = "{} wasn't in the initiale configuration.\nUnknown state. Please reconfigure and reprepare the whole silo.".format(source)
 
137
            logging.error(message)
 
138
            silomanager.set_config_status(silo_config, "Can't publish: " + message)
 
139
            sys.exit(1)
 
140
        supposed_dest_version = get_previous_distro_version_from_config(source)
 
141
 
 
142
        # check source version
 
143
        version_in_src = packagemanager.get_current_version_for_series(source, series, dest=src_ppa)
 
144
        if version_in_src != version:
 
145
            logging.error("Version in {}/{} ({}) is not the last one prepared ({}) (direct upload?).".format(src_ppa.owner.name, src_ppa.name, version_in_src, version))
 
146
            src_version_check_fail = True
 
147
 
 
148
        published_packagelist[source] = version
 
149
 
 
150
        # check destination version
 
151
        message = "Previous available version ({}) is not the latest version anymore in [] ([]).\nPlease ensure that this version has been merged back in trunk and relaunch prepare or use ignore version destination option.".format(supposed_dest_version)
 
152
        if launchpadmanager.is_dest_ubuntu_archive(dest_link):
 
153
            # Check that we don't skip a version unintendly
 
154
            version_in_dest = packagemanager.get_current_version_for_series(source, series, dest=launchpadmanager.get_ubuntu_archive())
 
155
            if not version_in_dest == supposed_dest_version:
 
156
                logging.error(message.replace('[]', '{}').format("the archive", version_in_dest))
 
157
                dest_version_check_fail = True
 
158
            packagelist_to_copy.append("{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}".format(src_ppa_name, src_ppa_pocket, series, distro_pocket, series, source, version, version_in_dest, publisher_nickname))
 
159
 
 
160
            if source in silo_config["mps"]:
 
161
                branch_uri = os.path.join(base_path, source)
 
162
                lp_project_name = launchpadmanager.get_resource_from_token(silo_config["mps"][source][0]).target_branch.project.name
 
163
                proposed_branch = PROPOSED_URL.format(lp_project_name, series)
 
164
                logging.info("Pushing {} to {}".format(source, proposed_branch))
 
165
                if not branchhandling.push_to_branch(branch_uri, proposed_branch, overwrite=True):
 
166
                    logging.error("We couldn't push {} to {}".format(source, proposed_branch))
 
167
                    error_pushing_branch = True
 
168
                    continue
 
169
        else:
 
170
            version_in_dest = packagemanager.get_current_version_for_series(source, series, dest=dest_archive)
 
171
            if not version_in_dest == supposed_dest_version:
 
172
                logging.warning(message.replace('[]', '{}').format(dest_archive.name, version_in_dest))
 
173
                dest_version_check_fail = True
 
174
 
 
175
    if error_pushing_branch:
 
176
        message = "We had some branches that couldn't be pushed to their proposed location."
 
177
        logging.error(message)
 
178
        silomanager.set_config_status(silo_config, "Can't publish: " + message)
 
179
        sys.exit(1)
 
180
 
 
181
    # remove eventual already published projects:
 
182
    for source in tools.get_published_to_distro_projects():
 
183
        with ignored(ValueError):
 
184
            all_silo_projects.remove(source)
 
185
 
 
186
    if all_silo_projects:
 
187
        message = "Some projects ({}) that were in the silo configuration list were not built. ".format(", ".join(all_silo_projects))
 
188
        if args.ignoremissingprojects:
 
189
            logging.info(message + "The ignore missing projects flag was set.")
 
190
        else:
 
191
            logging.error(message + "Prepare either prepare the latest missing projects or use the ignore missing projects flag which will release the lock on them.")
 
192
            silomanager.set_config_status(silo_config, "Can't publish: " + message)
 
193
            sys.exit(1)
 
194
 
 
195
    if src_version_check_fail:
 
196
        message = "Some packages in the ppa are not at the latest version. Please rerun the prepare job, eventually only with that project."
 
197
        logging.error(message)
 
198
        silomanager.set_config_status(silo_config, "Can't publish: " + message)
 
199
        sys.exit(1)
 
200
 
 
201
    if dest_version_check_fail:
 
202
        if args.ignoreversiondestination:
 
203
            logging.info("Some version destination don't match the latest available in dest, but ignore flag was set.")
 
204
        else:
 
205
            message = "Not all versions when starting prepare are the latest at destination. Read above and either use ignore version destination option or rerun prepare"
 
206
            logging.error(message)
 
207
            silomanager.set_config_status(silo_config, "Can't publish: " + message)
 
208
            sys.exit(1)
 
209
 
 
210
    #  now ack the changes (if copy is needed)
 
211
    if not launchpadmanager.is_dest_ubuntu_archive(dest_link):
 
212
        for (source, version, tip_rev, branch) in all_packages_uploaded:
 
213
            dest_archive.copyPackage(from_archive=src_ppa, from_pocket=src_pocket, from_series=series,
 
214
                                     include_binaries=True, to_pocket=dest_pocket, to_series=series,
 
215
                                     source_name=source, version=version)
 
216
    # generate a file for distro that will be rsynced from another place
 
217
    if packagelist_to_copy:
 
218
        package_list_rsync_file = PACKAGE_LIST_RSYNC_FILENAME_FORMAT.format(os.path.abspath('').split(os.path.sep)[-1], series)
 
219
        logging.info("Writing {} for being able to rsync from main machine".format(package_list_rsync_file))
 
220
        with open(package_list_rsync_file, 'w') as f:
 
221
            f.write("\n".join(packagelist_to_copy))
 
222
 
 
223
    # mark now all packages as published
 
224
    for (source, version, tip_rev, branch) in all_packages_uploaded:
 
225
        mark_project_as_published(source, version)
 
226
    generate_xml_artefacts("Publisher", [], PUBLISHER_ARTEFACTS_FILENAME)
 
227
    if not silomanager.set_config_step(silo_config, SILO_PUBLISHED):
 
228
        message = "Couldn't save the step in the configuration file."
 
229
        logging.error(message)
 
230
        silomanager.set_config_status(silo_config, "Publication failed: " + message)
 
231
        sys.exit(1)
 
232
 
 
233
    silomanager.set_config_pkgversionlist(silo_config, published_packagelist)
 
234
    silomanager.set_config_status(silo_config, "Packages migrating to destination. Wait a couple of hours and click merge & clean", add_url=False)