← Back to branch summary

~larry-e-works/uci-engine/amqp-to-kombu

« back to all changes in this revision

Viewing changes to cupstream2distro/prepare-package

  • Committer: Francis Ginther
  • Date: 2014-06-10 20:42:46 UTC
  • mto: This revision was merged to the branch mainline in revision 571.
  • Revision ID: francis.ginther@canonical.com-20140610204246-b1bsrik7nlcolqy7
Import lp:cupstream2distro rev 605.

Show diffs side-by-side

added added

removed removed

Lines of Context:
cupstream2distro/prepare-package
 
1
#!/usr/bin/python
 
2
# -*- coding: utf-8 -*-
 
3
# Copyright (C) 2012 Canonical
 
4
#
 
5
# Authors:
 
6
#  Didier Roche
 
7
#
 
8
# This program is free software; you can redistribute it and/or modify it under
 
9
# the terms of the GNU General Public License as published by the Free Software
 
10
# Foundation; version 3.
 
11
#
 
12
# This program is distributed in the hope that it will be useful, but WITHOUT
 
13
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 
14
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
 
15
# details.
 
16
#
 
17
# You should have received a copy of the GNU General Public License along with
 
18
# this program; if not, write to the Free Software Foundation, Inc.,
 
19
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 
20
 
 
21
import argparse
 
22
import glob
 
23
import logging
 
24
import os
 
25
import sys
 
26
 
 
27
from cupstream2distro import branchhandling, packagemanager, launchpadmanager
 
28
from cupstream2distro.tools import generate_xml_artefacts, get_packaging_diff_filename, save_project_config
 
29
from cupstream2distro.settings import PREPARE_ARTEFACTS_FILENAME_FORMAT, BOT_DEBFULLNAME
 
30
 
 
31
 
 
32
if __name__ == '__main__':
 
33
 
 
34
    logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
 
35
 
 
36
    parser = argparse.ArgumentParser(description="Prepare package for a specific component to push to a ppa",
 
37
                                     epilog="All options can be set by the corresponding long option name env variables as well")
 
38
 
 
39
    parser.add_argument("-n", "--sourcename", help="Source package name")
 
40
    parser.add_argument("-b", "--branch", help="Branch to use as the upstream source")
 
41
    parser.add_argument("-s", "--series", help="Serie used to build the package")
 
42
    parser.add_argument("-V", "--series-version", help="set series versions")
 
43
    parser.add_argument("-p", "--ppa", help="PPA to publish this package to (for instance: 'ubuntu-unity/daily-build')")
 
44
    parser.add_argument("-d", "--destppa", help="Consider this destppa instead of only the distro for finale destination", default='')
 
45
    parser.add_argument("-f", "--force-rebuild", help="Force rebuilding even if there is nothing to release", action='store_true')
 
46
    parser.add_argument("--condition-force-rebuild", help="Only force rebuild if the conditional file is present (* and other blobs are allowed)", nargs='*')
 
47
    parser.add_argument("--skip-checks", help="Skip destination version and conditional checking and call force rebuild", action='store_true')
 
48
 
 
49
    args = parser.parse_args()
 
50
 
 
51
    source_package_name = args.sourcename
 
52
    branch = args.branch
 
53
    series = args.series
 
54
    series_version = args.series_version
 
55
    ppa = args.ppa
 
56
    if not source_package_name:
 
57
        source_package_name = os.getenv("sourcename")
 
58
    if not branch:
 
59
        branch = os.getenv("branch")
 
60
    if not series:
 
61
        series = os.getenv("series")
 
62
    if not series_version:
 
63
        series = os.getenv("series_version")
 
64
    if not ppa:
 
65
        ppa = os.getenv("ppa")
 
66
    instance_info = "for source: {}, branch: {}, series: {}".format(source_package_name, branch, series)
 
67
    force_rebuild = False
 
68
 
 
69
    if not source_package_name or not branch or not series or not ppa:
 
70
        logging.error("Missing compulsory environment variables (sourcename, branch, series, ppa) {}".format(instance_info))
 
71
        sys.exit(1)
 
72
 
 
73
    # Grab project branch
 
74
    logging.info("Branching {} to {}".format(branch, source_package_name))
 
75
    branchhandling.get_branch(branch, source_package_name)
 
76
    os.chdir(source_package_name)
 
77
 
 
78
    # Get changelog informations
 
79
    logging.info("Get changelog info")
 
80
    with open("debian/changelog") as changelog:
 
81
        last_upstream_rev = packagemanager.get_latest_upstream_bzr_rev(changelog, args.destppa)
 
82
    packaging_version_in_bzr = packagemanager.get_packaging_version()
 
83
 
 
84
    if (source_package_name != packagemanager.get_packaging_sourcename()):
 
85
        logging.error("Projectbranch doesn't specify the same source name than the packaging itself {}".format(instance_info))
 
86
        sys.exit(1)
 
87
 
 
88
    # Check version in ppa
 
89
    logging.info("Get previous version in ppa")
 
90
    version_in_ppa = packagemanager.get_current_version_for_series(source_package_name, series, ppa)
 
91
    if packagemanager.is_version1_higher_than_version2(version_in_ppa, packaging_version_in_bzr):
 
92
        logging.info("A version in the ppa ({}) is higher than the proposed version in bzr ({}) (previous tests/builds failing?). Basing on that one.".format(version_in_ppa, packaging_version_in_bzr))
 
93
        previous_packaging_version = version_in_ppa
 
94
    else:
 
95
        previous_packaging_version = packaging_version_in_bzr
 
96
 
 
97
    # Check version in distro
 
98
    logging.info("Get version available in distro")
 
99
    distro_version = packagemanager.get_current_version_for_series(source_package_name, series)
 
100
    artefacts_file = os.path.abspath(os.path.join('..', PREPARE_ARTEFACTS_FILENAME_FORMAT.format(source_package_name)))
 
101
 
 
102
    # if the target is the distro, we ensure we have all releases in trunk
 
103
    if not args.destppa:
 
104
        dest_archive = launchpadmanager.get_ubuntu_archive()
 
105
        previous_finaledest_version = distro_version
 
106
    # target is a ppa
 
107
    else:
 
108
        dest_archive = launchpadmanager.get_ppa(args.destppa)
 
109
        previous_finaledest_version = packagemanager.get_current_version_for_series(source_package_name, series, args.destppa)
 
110
        # TODO: if none, we should maybe take distro_version?
 
111
 
 
112
    # ensure that the previous release (in distro or in dest ppa) ended up in the vcs
 
113
    if args.skip_checks:
 
114
        logging.info("Skip destination version and conditional checking requested. Will overwrite even if destination has a newer version or the (optional) condition isn't met.")
 
115
        force_rebuild = True
 
116
    else:
 
117
        with open("debian/changelog") as changelog:
 
118
            if not packagemanager.is_version_in_changelog(previous_finaledest_version, changelog):
 
119
                message = ("A version ({}) is available at the destination for that component but is not in trunk which is still at {}. "
 
120
                           "Ignoring that component {}.".format(previous_finaledest_version, packaging_version_in_bzr, instance_info))
 
121
                logging.info(message)
 
122
                generate_xml_artefacts("Prepare",  [message], artefacts_file)
 
123
                sys.exit(0)
 
124
 
 
125
    if force_rebuild or args.force_rebuild:
 
126
        logging.info("The rebuild of that component is forced.")
 
127
        force_rebuild = True
 
128
    if args.destppa:
 
129
        # if the version in the distro is higher than the previous packaging version (either the version in intermediate ppa or packaging bzr version
 
130
        # We are going to force releasing, but basing on this version
 
131
        if packagemanager.is_version1_higher_than_version2(distro_version, previous_packaging_version):
 
132
            logging.info("The version in distro ({}) is higher than what is proposed in the bzr version for that branch ({}). Basing on the distro version.".format(distro_version, previous_packaging_version))
 
133
            logging.info("Forcing release mode as the version in distro ({}) is higher than current destination ppa ({})".format(distro_version, previous_packaging_version))
 
134
            previous_packaging_version = distro_version
 
135
            force_rebuild = True
 
136
 
 
137
    # Check if we need to republish
 
138
    tip_bzr_rev = branchhandling.get_tip_bzr_revision()
 
139
    dest_source_package = None
 
140
    logging.info("Check if we need to release a new package")
 
141
    os.chdir('..')
 
142
    dest_source_package = packagemanager.get_source_package_from_dest(source_package_name, dest_archive, previous_finaledest_version, series)
 
143
    # TODO: all paths, on all projects, should have the stack dir as root, and only chdir in the script, not the functions.
 
144
    if dest_source_package:
 
145
        dest_source_package = os.path.abspath(dest_source_package)
 
146
    os.chdir(source_package_name)
 
147
    # nothing to release
 
148
    if not packagemanager.is_new_content_relevant_since_old_published_source(dest_source_package):
 
149
        if not force_rebuild:
 
150
            logging.info("No new useful revision published since last daily push, no need to upload this component")
 
151
            generate_xml_artefacts("Prepare",  [], artefacts_file)
 
152
            sys.exit(0)
 
153
        logging.info("No new useful revision published but force rebuilding is set")
 
154
        if not args.skip_checks and args.condition_force_rebuild:
 
155
            logging.info("Condition found without skip check, verifying that files exists")
 
156
            message = []
 
157
            for glob_filename in args.condition_force_rebuild:
 
158
                glob_found = False
 
159
                for filename in glob.glob(os.path.join("..", glob_filename)):
 
160
                    if os.path.isfile(filename):
 
161
                        glob_found = True
 
162
                if not glob_found:
 
163
                    msg = "{} doesn't exist".format(glob_filename)
 
164
                    logging.info(msg)
 
165
                    message.append(msg)
 
166
            # exit 0 as the precondition isn't there
 
167
            if message:
 
168
                generate_xml_artefacts("Prepare",  [], artefacts_file)
 
169
                sys.exit(0)
 
170
 
 
171
    packaging_version = packagemanager.create_new_packaging_version(previous_packaging_version, destppa=args.destppa, series_version=series_version)
 
172
 
 
173
    # Refresh symbol files
 
174
    logging.info("Update symbol files version if needed")
 
175
    packagemanager.refresh_symbol_files(packaging_version)
 
176
 
 
177
    # Prepare changelog
 
178
    logging.info("Prepare changelog content")
 
179
    log_content = branchhandling.return_log_diff(last_upstream_rev + 1)
 
180
    alreadyfixed_bugs = set()
 
181
    with open("debian/changelog") as f:
 
182
        alreadyfixed_bugs = packagemanager.collect_bugs_in_changelog_until_latest_snapshot(f, source_package_name)
 
183
    (authors_commits, bugs_set) = branchhandling.collect_author_commits(log_content, alreadyfixed_bugs)
 
184
    if force_rebuild:
 
185
        authors_commits[BOT_DEBFULLNAME] = ["New rebuild forced"]
 
186
    packagemanager.update_changelog(packaging_version, series, tip_bzr_rev, authors_commits, args.destppa)
 
187
 
 
188
    logging.info("Sync launchpad with current proposed work")
 
189
    branchhandling.commit_release(packaging_version, tip_bzr_rev)
 
190
    launchpadmanager.open_bugs_for_source(bugs_set, source_package_name, series)
 
191
 
 
192
    logging.info("Build source package")
 
193
    packagemanager.build_source_package(series, previous_finaledest_version, args.destppa)
 
194
    os.chdir('..')
 
195
    newdsc_path = "{}_{}.dsc".format(source_package_name, packaging_version.split(':')[-1])
 
196
 
 
197
    # second diff step: in case the source package was created from nothing (like a diff between trunk and the source package, but none after newly .dsc created extraction)
 
198
    if not force_rebuild and dest_source_package:
 
199
        logging.info("Check that the newly created source package has relevant diff")
 
200
        if not packagemanager.is_relevant_source_diff_from_previous_dest_version(newdsc_path, dest_source_package):
 
201
            message = ("There was a diff between trunk and the source package in ubuntu triggering a source package build. However, after that one was created, we diffed again "
 
202
                       "between that source in ubuntu and the newly created source extracted and found nothing relevant. Not upload that one then. Please check that the relevant "
 
203
                       "diff in the package creation is justified by diffing manually trunk and the source in ubuntu.")
 
204
            logging.info(message)
 
205
            generate_xml_artefacts("Prepare",  [message], artefacts_file)
 
206
            sys.exit(0)
 
207
 
 
208
    # We don't do package diff detection for a feature ppa
 
209
    if not args.destppa:
 
210
        # Prepare if a packaging change is part of the diff
 
211
        # FIXME: should be done with debdiff between all and new package
 
212
        logging.info("Generate a diff if meaningfull packaging changes")
 
213
        diff_filepath = get_packaging_diff_filename(source_package_name, packaging_version)
 
214
        olddsc_path = None
 
215
        if dest_source_package:
 
216
            for file in os.listdir(os.path.dirname(dest_source_package)):
 
217
                if file.endswith(".dsc"):
 
218
                    olddsc_path = os.path.join(os.path.dirname(dest_source_package), file)
 
219
        packagemanager.generate_diff_between_dsc(diff_filepath, olddsc_path, newdsc_path)
 
220
 
 
221
    logging.info("Upload to the ppa")
 
222
    packagemanager.upload_package(source_package_name, packaging_version, ppa)
 
223
 
 
224
    # Save infos for other jobs
 
225
    save_project_config(source_package_name, branch, tip_bzr_rev, previous_finaledest_version, packaging_version)
 
226
 
 
227
    # generate artefacts
 
228
    os.chdir(source_package_name)
 
229
    generate_xml_artefacts("Prepare",  [], artefacts_file)
 
230
 
 
231
# TODO: clean the ppa (remove all old unpublished packages)