4
* Uses xUnit (http://xunit.codeplex.com/) to test C# code.
6
* Assumes that when modifying a file with a path like `SomeAssembly/MyFile.cs`,
7
* that the test assembly that verifies the functionality of `SomeAssembly` is
8
* located at `SomeAssembly.Tests`.
10
* @concrete-extensible
12
class XUnitTestEngine extends ArcanistUnitTestEngine {
14
protected $runtimeEngine;
15
protected $buildEngine;
16
protected $testEngine;
17
protected $projectRoot;
18
protected $xunitHintPath;
19
protected $discoveryRules;
22
* This test engine supports running all tests.
24
protected function supportsRunAllTests() {
29
* Determines what executables and test paths to use. Between platforms this
30
* also changes whether the test engine is run under .NET or Mono. It also
31
* ensures that all of the required binaries are available for the tests to
36
protected function loadEnvironment() {
37
$this->projectRoot = $this->getWorkingCopy()->getProjectRoot();
39
// Determine build engine.
40
if (Filesystem::binaryExists('msbuild')) {
41
$this->buildEngine = 'msbuild';
42
} else if (Filesystem::binaryExists('xbuild')) {
43
$this->buildEngine = 'xbuild';
45
throw new Exception('Unable to find msbuild or xbuild in PATH!');
48
// Determine runtime engine (.NET or Mono).
49
if (phutil_is_windows()) {
50
$this->runtimeEngine = '';
51
} else if (Filesystem::binaryExists('mono')) {
52
$this->runtimeEngine = Filesystem::resolveBinary('mono');
54
throw new Exception('Unable to find Mono and you are not on Windows!');
57
// Read the discovery rules.
58
$this->discoveryRules =
59
$this->getConfigurationManager()->getConfigFromAnySource(
60
'unit.csharp.discovery');
61
if ($this->discoveryRules === null) {
63
'You must configure discovery rules to map C# files '.
64
'back to test projects (`unit.csharp.discovery` in .arcconfig).');
67
// Determine xUnit test runner path.
68
if ($this->xunitHintPath === null) {
69
$this->xunitHintPath =
70
$this->getConfigurationManager()->getConfigFromAnySource(
71
'unit.csharp.xunit.binary');
73
$xunit = $this->projectRoot.DIRECTORY_SEPARATOR.$this->xunitHintPath;
74
if (file_exists($xunit) && $this->xunitHintPath !== null) {
75
$this->testEngine = Filesystem::resolvePath($xunit);
76
} else if (Filesystem::binaryExists('xunit.console.clr4.exe')) {
77
$this->testEngine = 'xunit.console.clr4.exe';
80
"Unable to locate xUnit console runner. Configure ".
81
"it with the `unit.csharp.xunit.binary' option in .arcconfig");
86
* Main entry point for the test engine. Determines what assemblies to build
87
* and test based on the files that have changed.
89
* @return array Array of test results.
91
public function run() {
93
$this->loadEnvironment();
95
if ($this->getRunAllTests()) {
96
$paths = id(new FileFinder($this->projectRoot))->find();
98
$paths = $this->getPaths();
101
return $this->runAllTests($this->mapPathsToResults($paths));
105
* Applies the discovery rules to the set of paths specified.
107
* @param array Array of paths.
108
* @return array Array of paths to test projects and assemblies.
110
public function mapPathsToResults(array $paths) {
112
foreach ($this->discoveryRules as $regex => $targets) {
113
$regex = str_replace('/', '\\/', $regex);
114
foreach ($paths as $path) {
115
if (preg_match('/'.$regex.'/', $path) === 1) {
116
foreach ($targets as $target) {
117
// Index 0 is the test project (.csproj file)
118
// Index 1 is the output assembly (.dll file)
119
$project = preg_replace('/'.$regex.'/', $target[0], $path);
120
$project = $this->projectRoot.DIRECTORY_SEPARATOR.$project;
121
$assembly = preg_replace('/'.$regex.'/', $target[1], $path);
122
$assembly = $this->projectRoot.DIRECTORY_SEPARATOR.$assembly;
123
if (file_exists($project)) {
124
$project = Filesystem::resolvePath($project);
125
$assembly = Filesystem::resolvePath($assembly);
127
// Check to ensure uniqueness.
129
foreach ($results as $existing) {
130
if ($existing['assembly'] === $assembly) {
138
'project' => $project,
139
'assembly' => $assembly,
151
* Builds and runs the specified test assemblies.
153
* @param array Array of paths to test project files.
154
* @return array Array of test results.
156
public function runAllTests(array $test_projects) {
157
if (empty($test_projects)) {
162
$results[] = $this->generateProjects();
163
if ($this->resultsContainFailures($results)) {
164
return array_mergev($results);
166
$results[] = $this->buildProjects($test_projects);
167
if ($this->resultsContainFailures($results)) {
168
return array_mergev($results);
170
$results[] = $this->testAssemblies($test_projects);
172
return array_mergev($results);
176
* Determine whether or not a current set of results contains any failures.
177
* This is needed since we build the assemblies as part of the unit tests, but
178
* we can't run any of the unit tests if the build fails.
180
* @param array Array of results to check.
181
* @return bool If there are any failures in the results.
183
private function resultsContainFailures(array $results) {
184
$results = array_mergev($results);
185
foreach ($results as $result) {
186
if ($result->getResult() != ArcanistUnitTestResult::RESULT_PASS) {
194
* If the `Build` directory exists, we assume that this is a multi-platform
195
* project that requires generation of C# project files. Because we want to
196
* test that the generation and subsequent build is whole, we need to
197
* regenerate any projects in case the developer has added files through an
198
* IDE and then forgotten to add them to the respective `.definitions` file.
199
* By regenerating the projects we ensure that any missing definition entries
200
* will cause the build to fail.
202
* @return array Array of test results.
204
private function generateProjects() {
205
// No "Build" directory; so skip generation of projects.
206
if (!is_dir(Filesystem::resolvePath($this->projectRoot.'/Build'))) {
210
// No "Protobuild.exe" file; so skip generation of projects.
211
if (!is_file(Filesystem::resolvePath(
212
$this->projectRoot.'/Protobuild.exe'))) {
217
// Work out what platform the user is building for already.
218
$platform = phutil_is_windows() ? 'Windows' : 'Linux';
219
$files = Filesystem::listDirectory($this->projectRoot);
220
foreach ($files as $file) {
221
if (strtolower(substr($file, -4)) == '.sln') {
222
$parts = explode('.', $file);
223
$platform = $parts[count($parts) - 2];
228
$regenerate_start = microtime(true);
229
$regenerate_future = new ExecFuture(
230
'%C Protobuild.exe --resync %s',
231
$this->runtimeEngine,
233
$regenerate_future->setCWD(Filesystem::resolvePath(
234
$this->projectRoot));
236
$result = new ArcanistUnitTestResult();
237
$result->setName("(regenerate projects for $platform)");
240
$regenerate_future->resolvex();
241
$result->setResult(ArcanistUnitTestResult::RESULT_PASS);
242
} catch(CommandException $exc) {
243
if ($exc->getError() > 1) {
246
$result->setResult(ArcanistUnitTestResult::RESULT_FAIL);
247
$result->setUserdata($exc->getStdout());
250
$result->setDuration(microtime(true) - $regenerate_start);
251
$results[] = $result;
256
* Build the projects relevant for the specified test assemblies and return
257
* the results of the builds as test results. This build also passes the
258
* "SkipTestsOnBuild" parameter when building the projects, so that MSBuild
259
* conditionals can be used to prevent any tests running as part of the
260
* build itself (since the unit tester is about to run each of the tests
263
* @param array Array of test assemblies.
264
* @return array Array of test results.
266
private function buildProjects(array $test_assemblies) {
267
$build_futures = array();
268
$build_failed = false;
269
$build_start = microtime(true);
271
foreach ($test_assemblies as $test_assembly) {
272
$build_future = new ExecFuture(
275
'/p:SkipTestsOnBuild=True');
276
$build_future->setCWD(Filesystem::resolvePath(
277
dirname($test_assembly['project'])));
278
$build_futures[$test_assembly['project']] = $build_future;
280
$iterator = Futures($build_futures)->limit(1);
281
foreach ($iterator as $test_assembly => $future) {
282
$result = new ArcanistUnitTestResult();
283
$result->setName('(build) '.$test_assembly);
287
$result->setResult(ArcanistUnitTestResult::RESULT_PASS);
288
} catch(CommandException $exc) {
289
if ($exc->getError() > 1) {
292
$result->setResult(ArcanistUnitTestResult::RESULT_FAIL);
293
$result->setUserdata($exc->getStdout());
294
$build_failed = true;
297
$result->setDuration(microtime(true) - $build_start);
298
$results[] = $result;
304
* Build the future for running a unit test. This can be overridden to enable
305
* support for code coverage via another tool.
307
* @param string Name of the test assembly.
308
* @return array The future, output filename and coverage filename
309
* stored in an array.
311
protected function buildTestFuture($test_assembly) {
312
// FIXME: Can't use TempFile here as xUnit doesn't like
313
// UNIX-style full paths. It sees the leading / as the
314
// start of an option flag, even when quoted.
315
$xunit_temp = Filesystem::readRandomCharacters(10).'.results.xml';
316
if (file_exists($xunit_temp)) {
319
$future = new ExecFuture(
321
trim($this->runtimeEngine.' '.$this->testEngine),
324
$folder = Filesystem::resolvePath($this->projectRoot);
325
$future->setCWD($folder);
326
$combined = $folder.'/'.$xunit_temp;
327
if (phutil_is_windows()) {
328
$combined = $folder.'\\'.$xunit_temp;
330
return array($future, $combined, null);
334
* Run the xUnit test runner on each of the assemblies and parse the
337
* @param array Array of test assemblies.
338
* @return array Array of test results.
340
private function testAssemblies(array $test_assemblies) {
343
// Build the futures for running the tests.
346
$coverages = array();
347
foreach ($test_assemblies as $test_assembly) {
348
list($future_r, $xunit_temp, $coverage) =
349
$this->buildTestFuture($test_assembly['assembly']);
350
$futures[$test_assembly['assembly']] = $future_r;
351
$outputs[$test_assembly['assembly']] = $xunit_temp;
352
$coverages[$test_assembly['assembly']] = $coverage;
355
// Run all of the tests.
356
foreach (Futures($futures)->limit(8) as $test_assembly => $future) {
357
list($err, $stdout, $stderr) = $future->resolve();
359
if (file_exists($outputs[$test_assembly])) {
360
$result = $this->parseTestResult(
361
$outputs[$test_assembly],
362
$coverages[$test_assembly]);
363
$results[] = $result;
364
unlink($outputs[$test_assembly]);
366
// FIXME: There's a bug in Mono which causes a segmentation fault
367
// when xUnit.NET runs; this causes the XML file to not appear
368
// (depending on when the segmentation fault occurs). See
369
// https://bugzilla.xamarin.com/show_bug.cgi?id=16379
370
// for more information.
372
// Since it's not possible for the user to correct this error, we
373
// ignore the fact the tests didn't run here.
377
return array_mergev($results);
381
* Returns null for this implementation as xUnit does not support code
382
* coverage directly. Override this method in another class to provide code
383
* coverage information (also see @{class:CSharpToolsUnitEngine}).
385
* @param string The name of the coverage file if one was provided by
387
* @return array Code coverage results, or null.
389
protected function parseCoverageResult($coverage) {
394
* Parses the test results from xUnit.
396
* @param string The name of the xUnit results file.
397
* @param string The name of the coverage file if one was provided by
398
* `buildTestFuture`. This is passed through to
399
* `parseCoverageResult`.
400
* @return array Test results.
402
private function parseTestResult($xunit_tmp, $coverage) {
403
$xunit_dom = new DOMDocument();
404
$xunit_dom->loadXML(Filesystem::readFile($xunit_tmp));
407
$tests = $xunit_dom->getElementsByTagName('test');
408
foreach ($tests as $test) {
409
$name = $test->getAttribute('name');
410
$time = $test->getAttribute('time');
411
$status = ArcanistUnitTestResult::RESULT_UNSOUND;
412
switch ($test->getAttribute('result')) {
414
$status = ArcanistUnitTestResult::RESULT_PASS;
417
$status = ArcanistUnitTestResult::RESULT_FAIL;
420
$status = ArcanistUnitTestResult::RESULT_SKIP;
424
$reason = $test->getElementsByTagName('reason');
425
$failure = $test->getElementsByTagName('failure');
426
if ($reason->length > 0 || $failure->length > 0) {
427
$node = ($reason->length > 0) ? $reason : $failure;
428
$message = $node->item(0)->getElementsByTagName('message');
429
if ($message->length > 0) {
430
$userdata = $message->item(0)->nodeValue;
432
$stacktrace = $node->item(0)->getElementsByTagName('stack-trace');
433
if ($stacktrace->length > 0) {
434
$userdata .= "\n".$stacktrace->item(0)->nodeValue;
438
$result = new ArcanistUnitTestResult();
439
$result->setName($name);
440
$result->setResult($status);
441
$result->setDuration($time);
442
$result->setUserData($userdata);
443
if ($coverage != null) {
444
$result->setCoverage($this->parseCoverageResult($coverage));
446
$results[] = $result;
added
removed
src/unit/engine/XUnitTestEngine.php
