96
96
self.fileFinder = fileFinder
97
97
self.cacheDir = cacheDir
99
def getCacheFileName(self, buildName):
100
return os.path.join(self.cacheDir, "correct_hashes_" + buildName)
99
def getCacheFileName(self, jobName, buildName):
100
rootName = "correct_hashes_" + os.getenv("JENKINS_URL").split("/")[2].replace(":", "")
101
return os.path.join(self.cacheDir, rootName, jobName, buildName)
102
def getCachedFingerprint(self, buildName):
103
cacheFileName = self.getCacheFileName(buildName)
103
def getCachedFingerprint(self, jobName, buildName):
104
cacheFileName = self.getCacheFileName(jobName, buildName)
104
105
if os.path.isfile(cacheFileName):
105
106
return eval(open(cacheFileName).read())
107
def writeCache(self, buildName, updatedHashes):
108
cacheFileName = self.getCacheFileName(buildName)
108
def writeCache(self, jobName, buildName, updatedHashes):
109
cacheFileName = self.getCacheFileName(jobName, buildName)
110
from texttestlib import plugins
111
plugins.ensureDirExistsForFile(cacheFileName)
109
112
with open(cacheFileName, "w") as f:
110
113
pprint(updatedHashes, f)
132
135
self.jobRoot = jobRoot
133
136
self.verifier = FingerprintVerifier(fileFinder, cacheDir) if fileFinder else None
135
def findDifferences(self, jobName, build1, build2, verify):
138
def findDifferences(self, jobName, build1, build2):
136
139
buildsDir = getBuildsDir(self.jobRoot, jobName)
137
140
if not buildsDir:
139
fingerprint1 = self.getFingerprint(buildsDir, jobName, build1, verify)
140
fingerprint2 = self.getAndWaitForFingerprint(buildsDir, jobName, build2, verify)
142
fingerprint1 = self.getFingerprint(buildsDir, jobName, build1)
143
fingerprint2 = self.getAndWaitForFingerprint(buildsDir, jobName, build2)
141
144
if not fingerprint1 or not fingerprint2:
142
145
return [], bool(fingerprint2)
153
156
if isinstance(hash1, tuple):
155
158
if hash1 != hash2:
156
if verify and self.verifier:
157
160
correctedHash = self.verifier.getCorrectedHash(buildsDir, build2, file2, hash2)
158
161
if correctedHash:
159
162
hash2 = correctedHash
172
175
for artefact, (hash2, file2) in fingerprint2.items():
173
176
if artefact not in updatedHashes:
174
177
updatedHashes[artefact] = hash2
175
self.verifier.writeCache(build2, updatedHashes)
178
self.verifier.writeCache(jobName, build2, updatedHashes)
177
180
differences.sort()
178
181
return differences, True
189
192
print "Giving up waiting for fingerprints."
190
193
raise JobStillRunningException()
192
def getFingerprint(self, buildsDir, jobName, buildName, verify):
193
if verify and self.verifier:
194
cached = self.verifier.getCachedFingerprint(buildName)
195
def getFingerprint(self, buildsDir, jobName, buildName):
197
cached = self.verifier.getCachedFingerprint(jobName, buildName)
226
229
if author not in authors:
227
230
authors.append(author)
228
231
for msgNode in changeset.getElementsByTagName("msg"):
229
msg = msgNode.childNodes[0].nodeValue
230
self.addUnique(bugs, self.getBugs(msg))
232
if len(msgNode.childNodes):
233
msg = msgNode.childNodes[0].nodeValue
234
self.addUnique(bugs, self.getBugs(msg))
232
236
return authors, bugs
271
275
if "." in withoutEmail:
272
276
return " ".join([ part.capitalize() for part in withoutEmail.split(".") ])
274
return withoutEmail.encode("ascii", "xmlcharrefreplace")
279
return withoutEmail.encode("ascii", "xmlcharrefreplace")
281
return "unparseable author"
276
283
def addUnique(self, items, newItems):
277
284
for newItem in newItems:
278
285
if newItem not in items:
386
393
def findChanges(self, build1, build2):
388
markedChanges, projectChanges, fingerprintsFound = self.getChangesRecursively(self.jobName, build1, build2, verify=True)
395
markedChanges, projectChanges, fingerprintsFound = self.getChangesRecursively(self.jobName, build1, build2)
389
396
if not fingerprintsFound:
390
397
print "WARNING: tried to find Jenkins changes, but no fingerprints found for", self.jobName, "build", build2
391
398
if os.getenv("BUILD_ID") == "none" and os.getenv("BUILD_NUMBER") == build2:
402
409
changesFromMarking = [ self.getMarkChangeText(artefact, projectName, build1, build2) for artefact, projectName in markedChanges ]
403
410
return changesFromMarking + changesFromProjects
405
def getChangesRecursively(self, jobName, build1, build2, verify):
412
def getChangesRecursively(self, jobName, build1, build2):
406
413
# Find what artefacts have changed between times build
407
differences, fingerprintsFound = self.diffFinder.findDifferences(jobName, build1, build2, verify)
414
differences, fingerprintsFound = self.diffFinder.findDifferences(jobName, build1, build2)
408
415
# Organise them by project
409
416
markedChanges, differencesByProject = self.organiseByProject(differences)
410
417
# For each project, find out which builds were affected
411
418
projectChanges, recursiveChanges = self.getProjectChanges(differencesByProject)
412
419
for subProj, subBuild1, subBuild2 in recursiveChanges:
413
420
if subProj != jobName:
414
subMarkedChanges, subProjectChanges, _ = self.getChangesRecursively(subProj, subBuild1, subBuild2, verify=False)
421
subMarkedChanges, subProjectChanges, _ = self.getChangesRecursively(subProj, subBuild1, subBuild2)
415
422
for subMarkChange in subMarkedChanges:
416
423
if subMarkChange not in markedChanges:
417
424
markedChanges.append(subMarkChange)
480
487
regex = re.compile(artefact)
481
488
version1 = BuildDocument.create(buildsDir, build1).getArtefactVersion(regex)
482
489
version2 = BuildDocument.create(buildsDir, build2).getArtefactVersion(regex)
490
projectNameForDisplay = os.path.basename(projectName)
483
491
if version1 == version2:
484
return projectName + " was updated", "", []
492
return projectNameForDisplay + " was updated", "", []
486
return projectName + " " + version2, "", []
494
return projectNameForDisplay + " " + version2, "", []
488
496
print "WARNING: Artefact version not found for: " + projectName + " build: " + build2
489
497
return "Artefact version not found", "", []
499
507
buildsDir = getBuildsDir(jobRoot, os.getenv("JOB_NAME"))
501
509
buildLink = os.path.join(buildsDir, build)
502
if os.path.exists(buildLink):
510
if os.path.islink(buildLink):
503
511
return os.readlink(buildLink)
505
513
def parseEnvAsList(varName):
518
526
if __name__ == "__main__":
519
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
527
localFile = os.path.abspath(__file__) # <root>/texttestlib/default/batch/jenkinschanges.py
528
rootDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(localFile))))
529
sys.path.insert(0, rootDir)
520
530
buildName = sys.argv[1]
521
531
if len(sys.argv) > 2:
522
532
prevBuildName = sys.argv[2]
524
534
prevBuildName = str(int(buildName) - 1)
525
535
pprint(getChanges(prevBuildName, buildName, parseEnvAsDict("BUG_SYSTEM_DATA"), parseEnvAsList("MARKED_ARTEFACTS"),
526
536
os.getenv("FILE_FINDER", ""), os.getenv("CACHE_DIRECTORY", os.getcwd())))
b'\\ No newline at end of file'
added
removed
texttestlib/default/batch/jenkinschanges.py
