~psivaa/uci-engine/lander-upstart-logs-to-srv

logging.error("You already tried to build everything. You need to either:\n - use the ignore step option (to rebuild all MPs)\n - use the \"only rebuild packages\" option (to rebuild only some components associated to a MP) \n - use the watch only option (to only check ppa status without any rebuild).")

silomanager.set_config_status(silo_config, "Can't build: prevent a whole rebuild as one full build was already successful. Is that on purpose? Please check available options.")

sys.exit(1)

# remove everything but the archived .project_ files

logging.info("Whole rebuild, ensuring that the silo is in a coherent state. Keep the published files")

for filename in os.listdir("."):

# don't remove archive file and config file

if not ".{}_".format(PROJECT_CONFIG_SUFFIX) in filename and not "config" in filename:

try:

shutil.rmtree(filename)

except OSError:

os.remove(filename)

# reset the status

if not silomanager.set_config_step(silo_config, SILO_EMPTY):

logging.error("Couldn't reset the step in the configuration file.")

silomanager.set_config_status(silo_config, "Build failed: couldn't reset status or parameters for job.")

sys.exit(1)

# detect if we have a prepare only source not in configuration

components_not_in_config = []

for source in args.rebuildpackages:

if source not in silo_config["mps"]:

components_not_in_config.append(source)

if components_not_in_config:

message = "One or more components provided in rebuildpackages ({}) are not in the MP silo configuration. Ask for a reconfiguration of the silo with the new MPs to add please.".format(", ".join(components_not_in_config))

logging.error(message)

silomanager.set_config_status(silo_config, "Can't build: " + message)

sys.exit(1)

# cycle through all branches and prepare them

components_to_prepare = {}

for source in silo_config["mps"]:

if (not args.rebuildpackages and not args.watchonly) or source in args.rebuildpackages:

100

logging.info("Adding {} MP(s) to prepare".format(source))

101

for mp_link in silo_config['mps'][source]:

102

components_to_prepare.setdefault(source, []).append(launchpadmanager.get_resource_from_token(mp_link))

103

# cleaning (in case of partial rebuild)

104

clean_source(source)

105

106

# prepare source and dput by one

107

src_ppa_name = "{}/{}".format(src_ppa.owner.name, src_ppa.name)

108

dest_ppa_name = "{}/{}".format(dest.owner.name, dest.name)

109

110

src_pkgs_to_upload = {}

111

for source_package_name in components_to_prepare:

112

logging.info("=== Trying to prepare {} ===".format(source_package_name))

113

114

# get dest branch first

115

dest_branch = components_to_prepare[source_package_name][0].target_branch.web_link

116

logging.info("Branching {} to {}".format(dest_branch, source_package_name))

117

branchhandling.get_branch(dest_branch, source_package_name)

118

bzr_pending_branch_uri = os.path.abspath(source_package_name)

119

os.chdir(bzr_pending_branch_uri)

120

dest_tip_bzr_rev = branchhandling.get_tip_bzr_revision()

121

packaging_version_in_destbranch = packagemanager.get_packaging_version()

122

123

# first, try looking into the merge list and re-order it if there are any prereqs present

124

# if no prereqs are present, the components list stays the same

125

components = tools.reorder_branches_regarding_prereqs(components_to_prepare[source_package_name])

126

127

# get all branches one after another and merge them in bzr_pending_branch_uri

128

for mp in components:

129

branch = mp.source_branch.web_link

130

logging.info("Trying to merge {} (at rev {}) into {}".format(branch, mp.source_branch.revision_count, mp.target_branch.web_link))

131

132

bugs_fixes = ""

133

for bug in mp.getRelatedBugTasks():

134

if not bugs_fixes:

135

bugs_fixes = "Fixes: "

136

else:

137

bugs_fixes += ", "

138

bugs_fixes += bug.self_link.split('/')[-1]

139

140

commit_message = mp.commit_message

141

if commit_message:

142

commit_message = "{} {}".format(commit_message.encode('utf-8'), bugs_fixes)

143

144

# grab committers

145

committers = branchhandling.grab_committers_compared_to(bzr_pending_branch_uri, branch)

146

147

try:

148

if not branchhandling.merge_branch(bzr_pending_branch_uri, branch, commit_message, authors=committers):

149

logging.error("Conflicts when attempting to merge {} into previous merged + {}. You will need to reconcile manually branches that you provided as a parameter for merging into the destination branch and rerun this job".format(branch, mp.target_branch.web_link))

150

os.chdir('..')

151

silomanager.set_config_status(silo_config, "Can't build: Merge conflict.")

152

sys.exit(1)

153

# TODO: we should loop to show if there is more issues and then exit

154

except branchhandling.NoCommitFoundException as e:

155

message = "There is no commit message in {} and we can't get it from debian/changelog. Please check that you set a commit message on all your MPs if you didn't touch debian/changelog.".format(branch)

156

logging.error(message)

157

silomanager.set_config_status(silo_config, "Can't build: " + message)

158

sys.exit(1)

159

160

# Get changelog informations

161

logging.info("Get changelog info")

162

packaging_version_in_prepared_branch = packagemanager.get_packaging_version()

163

164

# Check version in ppa

165

logging.info("Get previous version in silo")

166

version_in_ppa = packagemanager.get_current_version_for_series(source_package_name, series.name, src_ppa_name)

167

if packagemanager.is_version1_higher_than_version2(version_in_ppa, packaging_version_in_prepared_branch):

168

logging.info("A version in the ppa ({}) is higher than the proposed version in bzr ({}) (previous tests/builds failing?). Basing on that one.".format(version_in_ppa, packaging_version_in_destbranch))

169

previous_packaging_version = version_in_ppa

170

else:

171

previous_packaging_version = packaging_version_in_prepared_branch

172

173

# Check version in distro

174

logging.info("Get version available in distro")

175

distro_version = packagemanager.get_current_version_for_series(source_package_name, series.name)

176

177

# if the target is the distro, we ensure we have all releases in proposed branch

178

if launchpadmanager.is_dest_ubuntu_archive(silo_config["global"]["dest"]):

179

previous_finaledest_version = distro_version

180

# target is a ppa

181

else:

182

previous_finaledest_version = packagemanager.get_current_version_for_series(source_package_name, series.name, dest=dest)

183

184

# ensure that the previous release (in distro or in dest ppa) ended up in the vcs

185

force_rebuild = False

186

if args.forcerebuild:

187

logging.info("Force rebuild option set. Will update even if destination don't contain latest version requested.")

188

force_rebuild = True

189

else:

190

with open("debian/changelog") as changelog:

191

if not packagemanager.is_version_in_changelog(previous_finaledest_version, changelog):

192

message = ("A version ({}) is available at the destination archive for that component but is not in the destination branch which is still at {}. You need to ensure that your version contains the fix in the destination or you can force rebuild to bypass the check.".format(previous_finaledest_version, packaging_version_in_prepared_branch))

193

logging.warning(message)

194

os.chdir('..')

195

silomanager.set_config_status(silo_config, "Can't build: Some projects are missing revisions from distribution.")

196

sys.exit(1)

197

198

if not launchpadmanager.is_dest_ubuntu_archive(silo_config["global"]["dest"]):

199

# if the version in the distro is higher than the previous packaging version (either the version in intermediate ppa or packaging bzr version

200

# We are going to force releasing, but basing on this version

201

if packagemanager.is_version1_higher_than_version2(distro_version, previous_packaging_version):

202

logging.info("The version in distro ({}) is higher than what is proposed in the bzr version for that branch ({}). Basing on the distro version.".format(distro_version, previous_packaging_version))

203

logging.info("Forcing release mode as the version in distro ({}) is higher than current destination ppa ({})".format(distro_version, previous_packaging_version))

204

previous_packaging_version = distro_version

205

force_rebuild = True

206

207

# Check if we need to republish

208

dest_source_package = None

209

logging.info("Check if we need to release a new package")

210

os.chdir('..')

211

dest_source_package = packagemanager.get_source_package_from_dest(source_package_name, dest, previous_finaledest_version, series.name)

212

if dest_source_package:

213

dest_source_package = os.path.abspath(dest_source_package)

214

os.chdir(bzr_pending_branch_uri)

215

# nothing to release

216

if not packagemanager.is_new_content_relevant_since_old_published_source(dest_source_package):

217

if not force_rebuild:

218

logging.info("No new useful revision published compared to dest, no need to upload this component")

219

os.chdir('..')

220

continue

221

logging.info("No new useful revision published but force rebuilding is set")

222

223

if launchpadmanager.is_dest_ubuntu_archive(silo_config["global"]["dest"]):

224

225

# only version if we don't see the flag to not version it.

226

if not packagemanager.has_dont_change_version_flag():

227

packaging_version = packagemanager.create_new_packaging_version(previous_packaging_version, series_version=SERIES_VERSION[series.name])

228

else:

229

packaging_version = packagemanager.get_packaging_version()

230

else:

231

packaging_version = packagemanager.create_new_packaging_version(previous_packaging_version, destppa=dest_ppa_name, series_version=SERIES_VERSION[series.name])

232

233

# Refresh symbol files

234

logging.info("Update symbol files version if needed")

235

packagemanager.refresh_symbol_files(packaging_version)

236

237

logging.info("Prepare changelog content")

238

authors_commits = {}

239

240

try:

241

# first try: in case someone release and release a version without merging back (we don't want to duplicate the commits). This is the main case anyway.

242

log_content = branchhandling.return_log_diff_from_tag(packaging_version_in_destbranch)

243

except:

244

try:

245

# second hammer: we then recollect from the last version in the destination (some lines can be duplicated)

246

log_content = branchhandling.return_log_diff_from_tag(previous_finaledest_version)

247

except:

248

# last hammer with FIXME: -100 is a hack as we don't have anymore any information on when the last release happen. People are transitionned to a tag system.

249

log_content = branchhandling.return_log_diff_since_last_release(branchhandling.return_log_diff("-100"))

250

alreadyfixed_bugs = set()

251

with open("debian/changelog") as f:

252

alreadyfixed_bugs = packagemanager.collect_bugs_in_changelog_until_latest_snapshot(f, source_package_name)

253

(authors_commits, bugs_set) = branchhandling.collect_author_commits(log_content, alreadyfixed_bugs)

254

if force_rebuild:

255

authors_commits.setdefault(BOT_DEBFULLNAME, []).append("New rebuild forced")

256

packagemanager.update_changelog(packaging_version, series.name, None, authors_commits)

257

branchhandling.commit_release(packaging_version)

258

259

logging.info("Sync launchpad with current proposed work")

260

launchpadmanager.open_bugs_for_source(bugs_set, source_package_name, series.name)

261

262

logging.info("Build source package")

263

packagemanager.build_source_package(series.name, previous_finaledest_version)

264

os.chdir('..')

265

newdsc_path = "{}_{}.dsc".format(source_package_name, packaging_version.split(':')[-1])

266

267

# second diff step: in case the source package was created from nothing (like a diff between trunk and the source package, but none after newly .dsc created extraction)

268

if not force_rebuild and dest_source_package:

269

logging.info("Check that the newly created source package has relevant diff")

270

if not packagemanager.is_relevant_source_diff_from_previous_dest_version(newdsc_path, dest_source_package):

271

message = ("There was a diff between trunk and the source package in ubuntu triggering a source package build. However, after that one was created, we diffed again "

272

"between that source in ubuntu and the newly created source extracted and found nothing relevant. Not upload that one then. Please check that the relevant "

273

"diff in the package creation is justified by diffing manually trunk and the source in ubuntu.")

274

logging.warning(message)

275

276

# check the .changes file: that we don't have an empty changelog

277

logging.info("Checking finale changelog")

278

with open("{}_{}_source.changes".format(source_package_name, packaging_version.split(':')[-1])) as f:

279

for line in f:

280

if re.match(" \* *$", line):

281

message = ("You generated an emtpy changelog content for {}. This generally means that one or many merge proposals have a change in debian/changelog with no UNRELEASED or no content in the changelog. "

282

"If you modify debian/changelog in your commit, the commit message for the merge proposal isn't used, so you need to put yourself some changelog content into that file. "

283

"Please change those MP and rerun the build job.".format(source_package_name))

284

logging.error(message)

285

silomanager.set_config_status(silo_config, "Build failed: Some packages generate an empty changelog. Check the job for more info.")

286

sys.exit(1)

287

288

289

# Prepare if a packaging change is part of the diff

290

logging.info("Generate a diff if meaningfull packaging changes")

291

diff_filepath = tools.get_packaging_diff_filename(source_package_name, packaging_version)

292

olddsc_path = None

293

if dest_source_package:

294

for file in os.listdir(os.path.dirname(dest_source_package)):

295

if file.endswith(".dsc"):

296

olddsc_path = os.path.join(os.path.dirname(dest_source_package), file)

297

packagemanager.generate_diff_between_dsc(diff_filepath, olddsc_path, newdsc_path)

298

# Save infos for uploading later on.

299

src_pkgs_to_upload[source_package_name] = (packaging_version, branch, dest_tip_bzr_rev, previous_finaledest_version)

300

301

302

# do the actual upload to the ppas

303

logging.info("Uploading components to the ppa")

304

for source_package_name in src_pkgs_to_upload:

305

(packaging_version, branch, dest_tip_bzr_rev, previous_finaledest_version) = src_pkgs_to_upload[source_package_name]

306

packagemanager.upload_package(source_package_name, packaging_version, src_ppa_name)

307

# Save infos for other jobs

308

tools.save_project_config(source_package_name, branch, dest_tip_bzr_rev, previous_finaledest_version, packaging_version)

309

310

311

# look at the ppa and generate .projects

312

logging.info("Looking at the ppa {} ({}) content".format(src_ppa.web_link, series.name))

313

# as we recreate all .projects file for those direct uploads, let's clean their metadata

314

for source in silo_config["sources"]:

315

clean_source(source)

316

direct_upload_not_in_list = []

317

try:

318

for pkg in src_ppa.getPublishedSources(distro_series=series, status="Published"):

319

if pkg.source_package_name in silo_config["mps"]:

320

continue

321

if pkg.source_package_name in silo_config["sources"]:

322

dest_current_version = packagemanager.get_current_version_for_series(pkg.source_package_name, series.name, dest=dest)

323

# if a direct source pacakge upload was already published, ignore it

324

published_sources = tools.get_published_to_distro_projects()

325

try:

326

if dest_current_version in published_sources[pkg.source_package_name]:

327

continue

328

except KeyError:

329

# scan the unpublished upload

330

tools.save_project_config(pkg.source_package_name, "", "", dest_current_version, pkg.source_package_version)

331

# download as well the .dsc file for watch-ppa

332

source_dir = packagemanager.get_source_package_from_dest(pkg.source_package_name, src_ppa, pkg.source_package_version, series.name)

333

shutil.copy(glob.glob(os.path.join(os.path.dirname(source_dir), '*.dsc'))[0], os.path.abspath('.'))

334

else:

335

direct_upload_not_in_list.append(pkg.source_package_name)

336

except lazr.restfulclient.errors.BadRequest:

337

pass # no package in the ppa yet

338

339

if direct_upload_not_in_list:

340

message = "Some packages ({}) that are not in the MP and not in sources list have been detected in the ppa. Please remove them from the ppa or ask for a reconfiguration.".format(", ".join(direct_upload_not_in_list))

341

logging.error(message)

342

silomanager.set_config_status(silo_config, "Build failed: " + message)

343

sys.exit(1)

344

345

# watch ppa

346

silomanager.set_config_status(silo_config, "Building", ping=False)

347

logging.info("Check ppa status in {}".format(src_ppa.web_link))

348

watch_ppa_exec = os.path.join(os.path.dirname(__file__), "watch-ppa")

349

call_params = [watch_ppa_exec, "-s", series.name, "-p", "{}/{}".format(src_ppa.owner.name, src_ppa.name)]

350

if not launchpadmanager.is_dest_ubuntu_archive(silo_config["global"]["dest"]):

351

call_params.extend(["--destppa", dest_ppa_name])

352

if subprocess.call(call_params) != 0:

353

silomanager.set_config_status(silo_config, "Build failed: Some packages failed to build.")

354

sys.exit(1)

Older »