Better tolerance for bad Sauce resultsorigin/saucefix

4 files changed, 115 insertions, 55 deletions

diff --git a/releng/org.eclipse.orion.client.releng/test/Gruntfile.js b/releng/org.eclipse.orion.client.releng/test/Gruntfile.js
index 14cc196..cc66449 100644
--- a/releng/org.eclipse.orion.client.releng/test/Gruntfile.js
+++ b/releng/org.eclipse.orion.client.releng/test/Gruntfile.js
@@ -43,9 +43,11 @@ module.exports = function(grunt) {
 	    nodeUrl = require("url"),
 	    archiver = require("archiver"),
 	    fmt = require("util").format,
+	    Q = require("q"),
 	    nodeutil = require("util"),
 	    zlib = require("zlib"),
-	    util = require(orionClient + "modules/orionode/build/utils")(grunt);
+	    util = require(orionClient + "modules/orionode/build/utils")(grunt),
+	    helpers = require("./test-helpers");
 
 	var config = util.loadBuildConfig(orionClient + "/releng/org.eclipse.orion.client.releng/builder/scripts/orion.build.js"),
 	    bundles = util.parseBundles(config, { orionClient: orionClient }),
@@ -151,26 +153,7 @@ module.exports = function(grunt) {
 	grunt.registerTask("test", ["server", "sauce"]);
 	grunt.registerTask("default", "test");
 
-	/**
-	 * For Hudson to parse out nice packages instead of (root), we have to add classname="packageName.className"
-	 * to the <testsuite> element, and prefix the "packageName." onto every <testcase>'s @classname. We also strip
-	 * out some problematic characters from the original classnames: [#?.]
-	 * @param {String} xml The xunit test result
-	 * @returns {String} The test result, fixed up
-	 */
-	function nicerName(xml, sauceResult, testUrl) {
-		function sanitize(s) {
-			return s.replace(/[^A-Za-z0-9_\.]/g, "_");
-		}
-		testUrl = testUrl.replace(/(^\/)|(\.html$)/g, "");
-		var platform = sanitize(sauceResult.platform.join(" ")),
-		    packageName = sanitize(fmt("%s.%s", platform, testUrl));
-		return xml
-			.replace(/(<testsuite\s+name="[^"]+")/g, fmt("$1 classname=\"%s\"", packageName))
-			.replace(/<testcase classname="([^"]+)"/g, function(match, className) {
-				return fmt("<testcase classname=\"%s.%s\"", packageName, className.replace(/[#?.]/g, "_"));
-			});
-	}
+
 
 	/**
 	 * Called per browser, per test page, after a test job is complete. Not called for a job that times out.
@@ -178,42 +161,52 @@ module.exports = function(grunt) {
 	 * @param {Function} callback Async CB to be invoked as callback(err, passOrFail) when done
 	 */
 	function onTestComplete(sauceResult, callback) {
-		function error(msgOrError) {
-			var e = nodeutil.isError(msgOrError) ? msgOrError : new Error(msgOrError);
-			grunt.warn(e && e.stack);
-			callback(e);
-		}
 		grunt.verbose.write("Got test result: ");
-		grunt.verbose.oklns(JSON.stringify(sauceResult));
+		grunt.verbose.write(sauceResult);
 		var mochaResult = sauceResult.result,
-		   id = sauceResult.id;
-		if (!mochaResult)
-			throw new Error(fmt("Test %s is missing 'result' field in response:\n%s", id, JSON.stringify(sauceResult)));
-		var testurl = mochaResult.url || "",
-		   gzippedXml = mochaResult.xunit,
-		   filename = fmt("TEST-%s_%s.xml", testurl.replace(/[^A-Za-z0-9_\-]/g, "_"), id);
-		if (/experienced an error/.test(sauceResult.message))
-			error(sauceResult.message);
-		if (!testurl)
-			error(fmt("Test %s did not return its url. Ensure it is using sauce.js", id));
-		if (!gzippedXml)
-			error(fmt("Test %s did not return an xunit result. Ensure it is using sauce.js.", testurl));
-
-		grunt.verbose.write("Inflating compressed xunit result...");
-		zlib.gunzip(new Buffer(gzippedXml, "base64"), function(e, buffer) {
-			if (e) {
-				error(e);
+		    id = sauceResult.id,
+		    testPageUrl = sauceResult.testPageUrl,
+		    filename = fmt("TEST-%s_%s.xml", testPageUrl.replace(/[^A-Za-z0-9_\-]/g, "_"), id);
+		Q.try(function() {
+			function throwError(msgOrError) {
+				var e = nodeutil.isError(msgOrError) ? msgOrError : new Error(msgOrError);
+				throw e;
 			}
-			grunt.verbose.ok();
-			var xunitReport = buffer.toString("utf8");
-			grunt.verbose.write("Replacing xunit testsuite name...");
-			xunitReport = nicerName(xunitReport, sauceResult, testurl);
-			grunt.verbose.ok();
+			if (!mochaResult)
+				throwError(fmt("Test %s is missing 'result' field in response:\n%s", id, JSON.stringify(sauceResult)));
+
+			var testurl = testPageUrl || mochaResult.url,
+			   gzippedXml = mochaResult.xunit;
+			if (/experienced an error/.test(sauceResult.message))
+				throwError(sauceResult.message);
+			if (!testurl)
+				throwError(fmt("Test %s did not return its url. Ensure it is using sauce.js", id));
+			if (!gzippedXml)
+				throwError(fmt("Test %s did not return an xunit result. Ensure it is using sauce.js.", testurl));
+
+			grunt.verbose.write("Inflating compressed xunit result...");
+			var deferred = Q.defer();
+			var zipresolver = deferred.makeNodeResolver(); // will resolve or reject `deferred`
+			zlib.gunzip(new Buffer(gzippedXml, "base64"), zipresolver);
+			return deferred.promise.then(function(buffer) {
+				grunt.verbose.ok();
+				var xunitReport = buffer.toString("utf8");
+				grunt.verbose.write("Replacing xunit testsuite name...");
+				xunitReport = helpers.xunit_cleanup(xunitReport, sauceResult, testurl);
+				grunt.verbose.ok();
+
+				helpers.xunit_write(grunt, nodePath.join(results, filename), xunitReport);
+				testFilenames.push(filename);
+				callback(undefined, true /*job pass*/);
+			});
+		})
+		.catch(function(error) {
+			// Something broke the test suite; output a generic xunit showing that it error'd
+			grunt.warn(error && error.stack);
 
-			grunt.file.write(nodePath.join(results, filename), xunitReport);
+			helpers.xunit_write(grunt, nodePath.join(results, filename), helpers.xunit_suite_error(filename, error));
 			testFilenames.push(filename);
-			grunt.verbose.ok();
-			callback(undefined, true /*job pass*/);
+			callback(error); /*job fail*/
 		});
 	}
 
diff --git a/releng/org.eclipse.orion.client.releng/test/package.json b/releng/org.eclipse.orion.client.releng/test/package.json
index be22595..4a65d80 100644
--- a/releng/org.eclipse.orion.client.releng/test/package.json
+++ b/releng/org.eclipse.orion.client.releng/test/package.json
@@ -20,7 +20,8 @@
     "grunt-cli": "~0.1.13",
     "grunt-contrib-connect": "~0.7.1",
     "grunt-curl": "~1.5.1",
-    "grunt-saucelabs": "~8.3.2"
+    "grunt-saucelabs": "~8.3.2",
+    "q": "~1.1.1"
   },
   "results": "./target/test-reports/",
   "urls": [
diff --git a/releng/org.eclipse.orion.client.releng/test/runtests.js b/releng/org.eclipse.orion.client.releng/test/runtests.js
index 0ae3848..35789e2 100644
--- a/releng/org.eclipse.orion.client.releng/test/runtests.js
+++ b/releng/org.eclipse.orion.client.releng/test/runtests.js
@@ -8,7 +8,7 @@
  * Contributors:
  *     IBM Corporation - initial API and implementation
  *******************************************************************************/
-/*jslint node:true*/
+/*eslint-env node*/
 /**
  * Performs the programatic equivalent of running `grunt --force --verbose` from the command line.
  * Need this for CF where running grunt from the command line has.. issues
@@ -23,7 +23,7 @@ try {
 		force: true,
 		stack: true,
 		verbose: true,
-		"no-color": !!(process.env.VCAP_APPLICATION) // CF logs can't deal with color codes
+		"no-color": ("VCAP_APPLICATION" in process.env) // CF logs can't deal with color codes
 	});
 } catch(e) {
 	// Uncaught exceptions are getting swallowed in CF env, need to log explicitly
diff --git a/releng/org.eclipse.orion.client.releng/test/test-helpers.js b/releng/org.eclipse.orion.client.releng/test/test-helpers.js
new file mode 100644
index 0000000..a811e9a
--- /dev/null
+++ b/releng/org.eclipse.orion.client.releng/test/test-helpers.js
@@ -0,0 +1,66 @@
+/*******************************************************************************
+ * Copyright (c) 2014 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials are made 
+ * available under the terms of the Eclipse Public License v1.0 
+ * (http://www.eclipse.org/legal/epl-v10.html), and the Eclipse Distribution 
+ * License v1.0 (http://www.eclipse.org/org/documents/edl-v10.html). 
+ *
+ * Contributors:
+ *     IBM Corporation - initial API and implementation
+ *******************************************************************************/
+/*eslint-env node*/
+var fmt = require("util").format;
+
+function sanitizeClassName(s) {
+	return s.replace(/[^A-Za-z0-9_\.]/g, "_");
+}
+
+function sanitizeXmlAttr(s) {
+	return s.replace(/['"&]/g, "_");
+}
+
+/**
+ * For Hudson to parse out nice packages instead of (root), we have to add classname="packageName.className"
+ * to the <testsuite> element, and prefix the "packageName." onto every <testcase>'s @classname. We also strip
+ * out some problematic characters from the original classnames: [#?.]
+ * @param {String} xml The xunit test result
+ * @returns {String} The test result, fixed up
+ */
+exports.xunit_cleanup = function(xml, sauceResult, testUrl) {
+	testUrl = testUrl.replace(/(^\/)|(\.html$)/g, "");
+	var platform = sanitizeClassName(sauceResult.platform.join(" ")),
+	    packageName = sanitizeClassName(fmt("%s.%s", platform, testUrl));
+	return xml
+		.replace(/(<testsuite\s+name="[^"]+")/g, fmt("$1 classname=\"%s\"", packageName))
+		.replace(/<testcase classname="([^"]+)"/g, function(match, className) {
+			return fmt("<testcase classname=\"%s.%s\"", packageName, className.replace(/[#?.]/g, "_"));
+		});
+};
+
+/**
+ * Returns a barebones xunit test suite mentioning the test url and error. This is useful for giving *something*
+ * to the Hudson build that indicates a failure. Otherwise unexpected errors might not be shown at all in the build.
+ * @returns {String} An xunit 
+ */
+exports.xunit_suite_error = function(testurl, error) {
+	var classname = sanitizeClassName(testurl),
+	    errorMessage = sanitizeXmlAttr(error.message);
+	var xml = ''
+		+ fmt('<testsuite name="%s" classname="%s" tests="1" failures="0" errors="1" skipped="0" timestamp="%s" time="0">', testurl, classname, (new Date()).toUTCString())
+			+ fmt('<testcase classname="SuiteFailure" name="SuiteFailure" time="0" message="%s">', errorMessage)
+				+ fmt('<failure classname="SuiteFailure" name="SuiteFailure" time="0" message="%s">', errorMessage)
+					+ '<![CDATA['
+					+ error.stack
+					+ ']]>'
+				+ '</failure>'
+			+ '</testcase>'
+		+ '</testsuite>';
+	return xml;
+};
+
+exports.xunit_write = function(grunt, filepath, contents) {
+	grunt.verbose.write(fmt("Writing result file %s", filepath));
+	grunt.file.write(filepath, contents);
+	grunt.verbose.ok();
+};
+