Merge pull request #1464 from tastejs/sjs/test-tweaks

⚡️ Automated Leak Detection on CI ⚡️ 

.jscsrc

@@ -29,6 +29,8 @@
 		"examples/vanilladart/**/*.js",
 		"examples/duel/www/**",
 		"examples/duel/src/main/webapp/js/lib/**",
+		"examples/ampersand/todomvc.bundle.js",
+		"examples/angular2/**/*.js",
 		"examples/polymer/elements/elements.build.js",
 		"examples/js_of_ocaml/js/*.js",
 	    "**/generated/**"

.travis.yml

@@ -31,12 +31,9 @@ before_install:
     - curl -Lo chrome.zip https://download-chromium.appspot.com/dl/Linux_x64 && unzip chrome.zip
     - curl -Lo chromedriver.zip http://chromedriver.storage.googleapis.com/2.18/chromedriver_linux64.zip && unzip chromedriver.zip
 before_script:
-    # install dependencies
-    - npm install -g gulp
     - cp site-assets/favicon.ico favicon.ico
     - python -m SimpleHTTPServer > /dev/null 2>&1 &
     - sleep 2
 script:
     # We want to gate on passing tests and a successful build
-    - gulp
     - ./test-runner.sh

test-runner.sh

@@ -11,6 +11,8 @@ get_changes ()
 
 if [ "$TRAVIS_BRANCH" = "master" ] && [ "$TRAVIS_PULL_REQUEST" = "false" ]
 then
+	npm i -g gulp
+	gulp
 	git submodule add -b gh-pages https://${GH_OAUTH_TOKEN}@github.com/${GH_OWNER}/${GH_PROJECT_NAME} site > /dev/null 2>&1
 	cd site
 	if git checkout gh-pages; then git checkout -b gh-pages; fi

tests/allTests.js

 'use strict';
 
 var testSuite = require('./test.js');
-var fs = require('fs');
 var argv = require('optimist').default('laxMode', false).default('browser', 'chrome').argv;
+var frameworkPathLookup = require('./framework-path-lookup');
 var rootUrl = 'http://localhost:8000/';
-var frameworkNamePattern = /^[a-z-_\d]+$/;
 
-var excludedFrameworks = [
-	// this implementation deviates from the specification to such an extent that they are
-	// not worth testing via a generic mechanism
-	'gwt',
-	// these implementations cannot be run offline, because they are hosted
-	'firebase-angular', 'meteor', 'socketstream',
-	// YUI is a special case here, it is not hosted, but fetches JS files dynamically
-	'yui',
-	// these frameworks take a long time to start-up, and there is no easy way to determine when they are ready
-	'cujo',
-	// sammyjs fails intermittently, it would appear that its state is sometimes updated asynchronously?
-	'sammyjs',
-	// elm-html batches UI updates with requestAnimationFrame which the tests
-	// don't wait for
-	'elm',
-	// these are examples that have been removed or are empty folders
-	'emberjs_require', 'dermis'
-];
+var list = frameworkPathLookup(argv.framework);
 
-// collect together the framework names from each of the subfolders
-var list = fs.readdirSync('../examples/')
-	.map(function (folderName) {
-		return { name: folderName, path: 'examples/' + folderName };
-	});
-
-// apps that are not hosted at the root of their folder need to be handled explicitly
-var exceptions = [
-	{ name: 'chaplin-brunch', path: 'examples/chaplin-brunch/public' },
-	{ name: 'angular-dart', path: 'examples/angular-dart/web' },
-	{ name: 'duel', path: 'examples/duel/www' },
-	{ name: 'vanilladart', path: 'examples/vanilladart/build/web' },
-	{ name: 'canjs_require', path: 'examples/canjs_require/' },
-	{ name: 'troopjs', path: 'examples/troopjs_require/' },
-	{ name: 'thorax_lumbar', path: 'examples/thorax_lumbar/public' }
-];
-list = list.map(function (framework) {
-	var exception = exceptions.filter(function (exFramework) {
-		return exFramework.name === framework.name;
-	});
-	return exception.length > 0 ? exception[0] : framework;
-});
-
-// filter out any folders that are not frameworks (.e.g  hidden files)
-list = list.filter(function (framework) {
-	return frameworkNamePattern.test(framework.name);
-});
-
-// filter out un-supported implementations
-list = list.filter(function (framework) {
-	return excludedFrameworks.indexOf(framework.name) === -1;
-});
-
-// if a specific framework has been named, just run this one
-if (argv.framework) {
-	list = list.filter(function (framework) {
-		return [].concat(argv.framework).some(function (f) {
-			return f === framework.name;
-		});
-	});
-
-	if (list.length === 0) {
+if (list.length === 0) {
 	console.log('You have either requested an unknown or an un-supported framework');
-	}
 }
 
 // run the tests for each framework

tests/framework-path-lookup.js

+var fs = require('fs');
+var frameworkNamePattern = /^[a-z-_\d]+$/;
+
+var excludedFrameworks = [
+	// this implementation deviates from the specification to such an extent that they are
+	// not worth testing via a generic mechanism
+	'gwt',
+	// these implementations cannot be run offline, because they are hosted
+	'firebase-angular', 'meteor', 'socketstream',
+	// YUI is a special case here, it is not hosted, but fetches JS files dynamically
+	'yui',
+	// these frameworks take a long time to start-up, and there is no easy way to determine when they are ready
+	'cujo',
+	// sammyjs fails intermittently, it would appear that its state is sometimes updated asynchronously?
+	'sammyjs',
+	// elm-html batches UI updates with requestAnimationFrame which the tests
+	// don't wait for
+	'elm',
+	// these are examples that have been removed or are empty folders
+	'emberjs_require', 'dermis'
+];
+
+module.exports = function (names) {
+	// collect together the framework names from each of the subfolders
+	var list = fs.readdirSync('../examples/')
+		.map(function (folderName) {
+			return { name: folderName, path: 'examples/' + folderName };
+		});
+
+	// apps that are not hosted at the root of their folder need to be handled explicitly
+	var exceptions = [
+		{ name: 'chaplin-brunch', path: 'examples/chaplin-brunch/public' },
+		{ name: 'angular-dart', path: 'examples/angular-dart/web' },
+		{ name: 'duel', path: 'examples/duel/www' },
+		{ name: 'vanilladart', path: 'examples/vanilladart/build/web' },
+		{ name: 'canjs_require', path: 'examples/canjs_require/' },
+		{ name: 'troopjs', path: 'examples/troopjs_require/' },
+		{ name: 'thorax_lumbar', path: 'examples/thorax_lumbar/public' }
+	];
+	list = list.map(function (framework) {
+		var exception = exceptions.filter(function (exFramework) {
+			return exFramework.name === framework.name;
+		});
+		return exception.length > 0 ? exception[0] : framework;
+	});
+
+	// filter out any folders that are not frameworks (.e.g  hidden files)
+	list = list.filter(function (framework) {
+		return frameworkNamePattern.test(framework.name);
+	});
+
+	// filter out un-supported implementations
+	list = list.filter(function (framework) {
+		return excludedFrameworks.indexOf(framework.name) === -1;
+	});
+
+	return list.filter(function (framework) {
+		return [].concat(names).some(function (f) {
+			return f === framework.name;
+		});
+	});
+}

tests/memory.js

+var drool = require('drool');
+var frameworkPathLookup = require('./framework-path-lookup');
+var argv = require('optimist').default('laxMode', false).default('browser', 'chrome').argv;
+var driverConfig = {
+	chromeOptions: 'no-sandbox'
+};
+
+if (typeof process.env.CHROME_PATH !== 'undefined') {
+	driverConfig.chromeBinaryPath = process.env.CHROME_PATH;
+}
+
+var driver = drool.start(driverConfig);
+var list = frameworkPathLookup(argv.framework);
+
+function idApp() {
+	return driver.findElement(drool.webdriver.By.css('#todoapp'))
+	.then(function () { return true; })
+	.thenCatch(function () { return false; });
+}
+
+function newTodoSelector(name) {
+	return idApp().then(function (isId) {
+		if (isId) {
+			return '#new-todo';
+		}
+
+		return '.new-todo';
+	});
+}
+
+function listSelector(name) {
+	return idApp().then(function (isId) {
+		if (isId) {
+			return '#todo-list li';
+		}
+
+		return '.todo-list li';
+	});
+}
+
+list.forEach(function (framework) {
+	drool.flow({
+		repeatCount: 5,
+		setup: function () {
+			driver.get('http://localhost:8000/' + framework.path + '/index.html');
+		},
+		action: function (name) {
+			driver.wait(function () {
+				return driver.findElement(drool.webdriver.By.css(newTodoSelector(name)))
+					.sendKeys('find magical goats', drool.webdriver.Key.ENTER)
+					.thenCatch(function () {
+						return false;
+					})
+				.then(function () {
+					return driver.findElement(drool.webdriver.By.css(listSelector(name))).isDisplayed()
+						.then(function () {
+							return true;
+						})
+				});
+			}, 10000);
+
+			driver.wait(function () {
+				return driver.findElement(drool.webdriver.By.css(listSelector(name))).click()
+					.thenCatch(function () {
+						return false;
+					})
+				.then(function () {
+					return true;
+				});
+			});
+
+			driver.findElement(drool.webdriver.By.css('.destroy')).click();
+		}.bind(null, framework.name),
+		assert: function (after, initial) {
+			var nodeIncrease = (after.nodes - initial.nodes);
+			var listenerIncrease = (after.jsEventListeners - initial.jsEventListeners);
+			console.log(this + ', ' +  nodeIncrease + ', ' +
+				(after.jsHeapSizeUsed - initial.jsHeapSizeUsed) + ', ' + listenerIncrease);
+
+			//https://code.google.com/p/chromium/issues/detail?id=516153
+			if (nodeIncrease > 5) {
+				throw new Error('Node Count leak detected!');
+			}
+
+			if (listenerIncrease > 0) {
+				throw new Error('Event Listener leak detected!');
+			}
+
+		}.bind(framework.name)
+	}, driver)
+});
+
+driver.quit();
+

tests/package.json

@@ -8,7 +8,8 @@
     "mocha": "*",
     "mocha-known-issues-reporter": "git+https://github.com/ColinEberhardt/mocha-known-issues-reporter.git#v0.0.0",
     "optimist": "^0.6.1",
-    "selenium-webdriver": "^2.46.1"
+    "selenium-webdriver": "^2.46.1",
+    "drool": "0.2.2"
   },
   "scripts": {
     "serve": "http-server -p 8000 ..",

tests/run.sh

@@ -3,4 +3,5 @@
 args="$@"
 
 npm i && \
+eval "node memory.js $args" && \
 eval "npm test -- $args"