Skip to content

Commit

Permalink
Refactoring, the task promise is rejected when any error occurs. Its …
Browse files Browse the repository at this point in the history
…main advantage is that the control flow has become much simpler, errors from multiple sources can be handled at a single point (the grunt task's entry point).

Errors caused by too big test results are correctly reported (axemclion#123).
Added new grunt tasks for testing test failures and too big test results.
  • Loading branch information
gvas committed May 26, 2014
1 parent 98bb0db commit e15b9c0
Show file tree
Hide file tree
Showing 5 changed files with 210 additions and 53 deletions.
51 changes: 48 additions & 3 deletions Gruntfile.js
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ module.exports = function(grunt) {
}
},
'saucelabs-mocha': {
all: {
succeeds: {
//username: '',
//key: '',
options: {
Expand All @@ -67,10 +67,24 @@ module.exports = function(grunt) {
'video-upload-on-pass': false
}
}
},
fails: {
options: {
urls: ['http://127.0.0.1:9999/mocha/test/browser/fails.html'],
build: process.env.TRAVIS_JOB_ID,
browsers: [{
browserName: 'googlechrome',
platform: 'XP'
}],
testname: 'fails',
sauceConfig: {
'video-upload-on-pass': false
}
}
}
},
'saucelabs-custom': {
all: {
succeeds: {
//username: '',
//key: '',
options: {
Expand All @@ -82,6 +96,34 @@ module.exports = function(grunt) {
'video-upload-on-pass': false
}
}
},
fails: {
options: {
urls: ['http://127.0.0.1:9999/custom/fails.html'],
build: process.env.TRAVIS_JOB_ID,
browsers: [{
browserName: 'googlechrome',
platform: 'XP'
}],
testname: 'fails',
sauceConfig: {
'video-upload-on-pass': false
}
}
},
'test-result-too-big': {
options: {
urls: ['http://127.0.0.1:9999/custom/test-result-too-big.html'],
build: process.env.TRAVIS_JOB_ID,
browsers: [{
browserName: 'googlechrome',
platform: 'XP'
}],
testname: 'test-result-too-big',
sauceConfig: {
'video-upload-on-pass': false
}
}
}
},
'saucelabs-qunit': {
Expand Down Expand Up @@ -126,10 +168,13 @@ module.exports = function(grunt) {

var testjobs = ['jshint', 'connect'];
if (typeof process.env.SAUCE_ACCESS_KEY !== 'undefined'){
testjobs = testjobs.concat(['saucelabs-qunit', 'saucelabs-jasmine', 'saucelabs-yui', 'saucelabs-mocha', 'saucelabs-custom']);
testjobs = testjobs.concat(['saucelabs-qunit', 'saucelabs-jasmine', 'saucelabs-yui', 'saucelabs-mocha:succeeds', 'saucelabs-custom:succeeds']);
}

grunt.registerTask("dev", ["connect", "watch"]);
grunt.registerTask('test', testjobs);
grunt.registerTask('default', ['test']);
grunt.registerTask('custom-fails', ['jshint', 'connect', 'saucelabs-custom:fails']);
grunt.registerTask('custom-test-result-too-big', ['jshint', 'connect', 'saucelabs-custom:test-result-too-big']);
grunt.registerTask('mocha-fails', ['jshint', 'connect', 'saucelabs-mocha:fails']);
};
122 changes: 72 additions & 50 deletions tasks/saucelabs.js
Original file line number Diff line number Diff line change
Expand Up @@ -11,26 +11,21 @@ module.exports = function(grunt) {
return result.passed;
},
qunit: function(result){
if (result.passed === undefined){ return undefined; }
return result.passed == result.total;
},
mocha: function(result){
if (result.passes === undefined){ return undefined; }
return result.failures === 0;
},
'YUI Test': function(result){
if (result.passed === undefined){ return undefined; }
return result.passed == result.total;
},
custom: function(result){
if (result.passed === undefined){ return undefined; }
return result.failed === 0;
}
};

var TestResult = function(jobId, user, key, framework, testInterval){
var url = 'https://saucelabs.com/rest/v1/' + user + '/js-tests/status';
var deferred = Q.defer();

var requestParams = {
method: 'post',
Expand All @@ -45,45 +40,51 @@ module.exports = function(grunt) {
}
};

var checkStatus = function () {
Q
function checkStatus () {
return Q
.nfcall(rqst, requestParams)
.then(
function (result) {
var body = result[1];
var testInfo = body['js tests'][0];
var testInfo;

if (testInfo.status == 'test error') {
throw 'Test Error';
if (!body || !body['js tests'] || !body['js tests'].length || !body['js tests'][0]) {
throw 'Unexpected response from the Sauce Labs API.';
}

if (!body.completed) {
setTimeout(checkStatus, testInterval);
} else {
testInfo.passed = testInfo.result ? resultParsers[framework](testInfo.result) : false;
deferred.resolve(testInfo);
if (body.completed) {
testInfo = body['js tests'][0];
if (!testInfo.result) {
// Sauce Labs' 64KB limit on custom-data was hit, see #123.
throw 'Test result is too big. Shrink it by modifying the test reporter or splitting the tests into multiple test runners.';
} else if (testInfo.status == 'test error') {
// A detailed error message should be composed here after
// the Sauce Labs API is modified to report errors better,
// see #102.
throw 'Test Error';
}
}

return body;
},
function (error) {
throw 'Error connecting to api to get test status: ' + error.toString();
}
)
.catch(function (error) {
// We indicate errors by setting the passed element to undefined
// instead of rejecting the deferred.
deferred.resolve({
passed: undefined,
result: {
message: error.toString()
}
});
})
.done();
};
.then(function (body) {
if (body.completed) {
var testInfo = body['js tests'][0];

checkStatus();
testInfo.passed = resultParsers[framework](testInfo.result);
return testInfo;
} else {
return Q
.delay(testInterval)
.thenResolve(checkStatus());
}
});
}

return deferred.promise;
return checkStatus();
};

var TestRunner = function(user, key, testInterval) {
Expand Down Expand Up @@ -120,16 +121,10 @@ module.exports = function(grunt) {
grunt.log.writeln("Warning: This url might use a port that is not proxied by Sauce Connect.".yellow);
}

if (result.passed === undefined) {
grunt.log.error(result.result.message);
} else {
grunt.log.writeln("Passed: %s", result.passed);
}
grunt.log.writeln("Passed: %s", result.passed);
grunt.log.writeln("Url %s", result.url);

return result;
}, function (e) {
grunt.log.error('some error? %s', e);
return result.passed;
});
});
}
Expand All @@ -147,9 +142,7 @@ module.exports = function(grunt) {

return Q.all(promises)
.then(function (results) {
return results.map(function (result) {
return result.passed;
});
return results.indexOf(false) === -1;
});
};

Expand Down Expand Up @@ -253,29 +246,58 @@ module.exports = function(grunt) {
callback(false);
return;
}
grunt.log.ok('Connected to Saucelabs');
grunt.log.ok('Connected to Sauce Labs');

test
.runTests(arg.browsers, arg.pages, framework, arg.identifier, arg.build, arg.testname, arg.sauceConfig, arg.onTestComplete, arg.throttled)
.then(function (status) {
status = status.every(function (passed) { return passed; });
grunt.log[status ? 'ok' : 'error']('All tests completed with status %s', status);
.then(function (passed) {
if (passed) {
grunt.log.ok('All tests completed with status true');
} else {
grunt.log.error('All tests completed with status false');
}
return passed;
})
.finally(function () {
var deferred = Q.defer();
grunt.log.writeln('=> Stopping Tunnel to Sauce Labs'.inverse.bold);
tunnel.stop(function () {
callback(status);
deferred.resolve();
});
return deferred.promise;
})
.then(
function(passed) {
callback(passed);
},
function(error) {
grunt.log.error(error.toString());
callback(false);
}
)
.done();
});

} else {
test
.runTests(arg.browsers, arg.pages, framework, null, arg.build, arg.testname, arg.sauceConfig, arg.onTestComplete, arg.throttled)
.then(function (status) {
status = status.every(function (passed) { return passed; });
grunt.log[status ? 'ok' : 'error']('All tests completed with status %s', status);
callback(status);
.then(function (passed) {
if (passed) {
grunt.log.ok('All tests completed with status true');
} else {
grunt.log.error('All tests completed with status false');
}
return passed;
})
.then(
function(passed) {
callback(passed);
},
function(error) {
grunt.log.error(error.toString());
callback(false);
}
)
.done();
}
}
Expand Down
21 changes: 21 additions & 0 deletions test/custom/fails.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
<!DOCTYPE html>
<html>
<head>
<title>Tests behavior when some of the tests fail</title>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<script>
window.onload = function () {
window.global_test_results = {
passed: 1,
failed: 1,
total: 2,
duration: 1000,
tests: ''
};
};
</script>
</head>
<body>
</body>
</html>
21 changes: 21 additions & 0 deletions test/custom/test-result-too-big.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
<!DOCTYPE html>
<html>
<head>
<title>Tests behavior when test result exceeds 64KB</title>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<script>
window.onload = function () {
window.global_test_results = {
passed: 1,
failed: 0,
total: 1,
duration: 1000,
tests: (function () { var foo = []; for (var i = 0; i <= Math.pow(2, 13); i++) { foo.push('01234567'); } return foo.join(''); }())
};
};
</script>
</head>
<body>
</body>
</html>
48 changes: 48 additions & 0 deletions test/mocha/test/browser/fails.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
<html>
<head>
<title>Tests behavior when some of the tests fail</title>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<link rel="stylesheet" href="../../mocha.css" />
<script src="../../mocha.js"></script>
<script>mocha.setup('bdd')</script>
<script>
describe('Something', function () {
it('should succeed', function () {
});

it('should succeed but fails', function () {
throw new Error('Fails.');
});
});
onload = function(){
var runner = mocha.run();

var failedTests = [];
runner.on('end', function(){
window.mochaResults = runner.stats;
window.mochaResults.reports = failedTests;
});

runner.on('fail', logFailure);

function logFailure(test, err){

var flattenTitles = function(test){
var titles = [];
while (test.parent.title){
titles.push(test.parent.title);
test = test.parent;
}
return titles.reverse();
};

failedTests.push({name: test.title, result: false, message: err.message, stack: err.stack, titles: flattenTitles(test) });
};
};
</script>
</head>
<body>
<div id="mocha"></div>
</body>
</html>

0 comments on commit e15b9c0

Please sign in to comment.