Merge pull request #814 from ColinEberhardt/improved-test-clarity

Improved test clarity

browser-tests/allTests.js

@@ -6,6 +6,10 @@ var argv = require('optimist').default('laxMode', false).argv;
 var rootUrl = 'http://localhost:8000/';
 var frameworkNamePattern = /^[a-z-_]+$/;
 
+// these implementations deviate from the specification to such an extent that they are 
+// not worth testing via a generic mecanism
+var excludedFrameworks = ['gwt', 'polymer'];
+
 // collect together the framework names from each of the subfolders
 var list = fs.readdirSync('../architecture-examples/')
 	.map(function (folderName) {
@@ -31,7 +35,6 @@ var list = fs.readdirSync('../architecture-examples/')
 var exceptions = [
 	{ name: 'chaplin-brunch', path: 'labs/dependency-examples/chaplin-brunch/public' }
 ];
-
 list = list.map(function (framework) {
 	var exception = exceptions.filter(function (exFramework) {
 		return exFramework.name === framework.name;
@@ -44,11 +47,20 @@ list = list.filter(function (framework) {
 	return frameworkNamePattern.test(framework.name);
 });
 
+// filter out un-supported implementations
+list = list.filter(function (framework) {
+	return excludedFrameworks.indexOf(framework.name) === -1;
+});
+
 // if a specific framework has been named, just run this one
 if (argv.framework) {
 	list = list.filter(function (framework) {
 		return framework.name === argv.framework;
 	});
+
+	if(list.length === 0) {
+		console.log('You have either requested an unknown or an un-supported framework');
+	}
 }
 
 // run the tests for each framework

browser-tests/knownIssues.js

@@ -37,62 +37,5 @@ module.exports = [
 	// for some reason the persistence test fails for knockout, even though persistence is working
 	// just fine. Perhaps there is something asynchronous going on that is causing the assert
 	// to be executed early?
-	'TodoMVC - knockoutjs, Persistence, should persist its data',
-
-
-	// ----------------- Unsupported implementations!! -----------
-
-	// the following are TodoMVC implementations that are not supported by the autoated UI
-	// tests, and as a result have numerous failures.
-
-	// polymer - does not follow the HTML spec
-	'TodoMVC - polymer, New Todo, should allow me to add todo items',
-	'TodoMVC - polymer, New Todo, should clear text input field when an item is added',
-	'TodoMVC - polymer, New Todo, should trim text input',
-	'TodoMVC - polymer, New Todo, should show #main and #footer when items added',
-	'TodoMVC - polymer, Mark all as completed, should allow me to mark all items as completed',
-	'TodoMVC - polymer, Mark all as completed, should allow me to clear the completion state of all items',
-	'TodoMVC - polymer, Mark all as completed, complete all checkbox should update state when items are completed / cleared',
-	'TodoMVC - polymer, Item, should allow me to mark items as complete',
-	'TodoMVC - polymer, Item, should allow me to un-mark items as complete',
-	'TodoMVC - polymer, Item, should allow me to edit an item',
-	'TodoMVC - polymer, Editing, should hide other controls when editing',
-	'TodoMVC - polymer, Editing, should save edits on enter',
-	'TodoMVC - polymer, Editing, should save edits on blur',
-	'TodoMVC - polymer, Editing, should trim entered text',
-	'TodoMVC - polymer, Editing, should remove the item if an empty text string was entered',
-	'TodoMVC - polymer, Editing, should cancel edits on escape',
-	'TodoMVC - polymer, Counter, should display the current number of todo items',
-	'TodoMVC - polymer, Clear completed button, should display the number of completed items',
-	'TodoMVC - polymer, Clear completed button, should remove completed items when clicked',
-	'TodoMVC - polymer, Clear completed button, should be hidden when there are no items that are completed',
-	'TodoMVC - polymer, Persistence, should persist its data',
-	'TodoMVC - polymer, Routing, should allow me to display active items',
-	'TodoMVC - polymer, Routing, should allow me to display completed items',
-	'TodoMVC - polymer, Routing, should allow me to display all items',
-	'TodoMVC - polymer, Routing, should highlight the currently applied filter',
-
-	// gwt - does not follow the HTML spec closely eough for testing
-	'TodoMVC - gwt, New Todo, should allow me to add todo items',
-	'TodoMVC - gwt, New Todo, should trim text input',
-	'TodoMVC - gwt, Mark all as completed, should allow me to mark all items as completed',
-	'TodoMVC - gwt, Mark all as completed, should allow me to clear the completion state of all items',
-	'TodoMVC - gwt, Mark all as completed, complete all checkbox should update state when items are completed / cleared',
-	'TodoMVC - gwt, Item, should allow me to mark items as complete',
-	'TodoMVC - gwt, Item, should allow me to un-mark items as complete',
-	'TodoMVC - gwt, Item, should allow me to edit an item',
-	'TodoMVC - gwt, Editing, should hide other controls when editing',
-	'TodoMVC - gwt, Editing, should save edits on enter',
-	'TodoMVC - gwt, Editing, should save edits on blur',
-	'TodoMVC - gwt, Editing, should trim entered text',
-	'TodoMVC - gwt, Editing, should remove the item if an empty text string was entered',
-	'TodoMVC - gwt, Editing, should cancel edits on escape',
-	'TodoMVC - gwt, Clear completed button, should display the number of completed items',
-	'TodoMVC - gwt, Clear completed button, should remove completed items when clicked',
-	'TodoMVC - gwt, Clear completed button, should be hidden when there are no items that are completed',
-	'TodoMVC - gwt, Persistence, should persist its data',
-	'TodoMVC - gwt, Routing, should allow me to display active items',
-	'TodoMVC - gwt, Routing, should allow me to display completed items',
-	'TodoMVC - gwt, Routing, should allow me to display all items',
-	'TodoMVC - gwt, Routing, should highlight the currently applied filter',
+	'TodoMVC - knockoutjs, Persistence, should persist its data'
 ];

browser-tests/test.js

@@ -194,8 +194,8 @@ module.exports.todoMVCTest = function (frameworkName, baseUrl, speedMode, laxMod
 				createStandardItems();
 				page.doubleClickItemAtIndex(1);
 
-				testOps.assertItemToggleIsHidden();
-				testOps.assertItemLabelIsHidden();
+				testOps.assertItemToggleIsHidden(1);
+				testOps.assertItemLabelIsHidden(1);
 			});
 
 			test.it('should save edits on enter', function () {

browser-tests/testOperations.js

@@ -9,36 +9,37 @@ function TestOperations(page) {
 	// element exists. The standard approach is to obtain an array of elements
 	// and test that the length is zero. In this case the item is hidden if
 	// it is either not in the DOM, or is in the DOM but not visible.
-	function testIsHidden(elements) {
+	function testIsHidden(elements, name) {
 		if (elements.length === 1) {
 			elements[0].isDisplayed().then(function (isDisplayed) {
-				assert(!isDisplayed);
+				assert(!isDisplayed, 'the ' + name + ' element should be hidden');
 			});
 		}
 	}
 
-	function testIsVisible(elements) {
+	function testIsVisible(elements, name) {
 		assert.equal(1, elements.length);
 		elements[0].isDisplayed().then(function (isDisplayed) {
-			assert(isDisplayed);
+			assert(isDisplayed, 'the ' + name + ' element should be displayed');
 		});
 	}
 
 	this.assertClearCompleteButtonIsHidden = function () {
 		page.tryGetClearCompleteButton().then(function (element) {
-			testIsHidden(element);
+			testIsHidden(element, 'clear completed items button');
 		});
 	};
 
 	this.assertClearCompleteButtonIsVisible = function () {
 		page.tryGetClearCompleteButton().then(function (element) {
-			testIsVisible(element);
+			testIsVisible(element, 'clear completed items button');
 		});
 	};
 
 	this.assertItemCount = function (itemCount) {
 		page.getItemElements().then(function (toDoItems) {
-			assert.equal(itemCount, toDoItems.length);
+			assert.equal(itemCount, toDoItems.length,
+				itemCount + ' items expected in the todo list, ' + toDoItems.length + ' items observed');
 		});
 	};
 
@@ -53,37 +54,38 @@ function TestOperations(page) {
 
 	this.assertMainSectionIsHidden = function () {
 		page.tryGetMainSectionElement().then(function (mainSection) {
-			testIsHidden(mainSection);
+			testIsHidden(mainSection, 'main');
 		});
 	};
 
 	this.assertFooterIsHidden = function () {
 		page.tryGetFooterElement().then(function (footer) {
-			testIsHidden(footer);
+			testIsHidden(footer, 'footer');
 		});
 	};
 
 	this.assertMainSectionIsVisible = function () {
 		page.tryGetMainSectionElement().then(function (mainSection) {
-			testIsVisible(mainSection);
+			testIsVisible(mainSection, 'main');
 		});
 	};
 
-	this.assertItemToggleIsHidden = function () {
-		page.tryGetToggleForItemAtIndex().then(function (toggleItem) {
-			testIsHidden(toggleItem);
+	//TODO: fishy!
+	this.assertItemToggleIsHidden = function (index) {
+		page.tryGetToggleForItemAtIndex(index).then(function (toggleItem) {
+			testIsHidden(toggleItem, 'item-toggle');
 		});
 	};
 
-	this.assertItemLabelIsHidden = function () {
-		page.tryGetItemLabelAtIndex().then(function (toggleItem) {
-			testIsHidden(toggleItem);
+	this.assertItemLabelIsHidden = function (index) {
+		page.tryGetItemLabelAtIndex(index).then(function (toggleItem) {
+			testIsHidden(toggleItem, 'item-label');
 		});
 	};
 
 	this.assertFooterIsVisible = function () {
 		page.tryGetFooterElement().then(function (footer) {
-			testIsVisible(footer);
+			testIsVisible(footer, 'footer');
 		});
 	};
 
@@ -95,14 +97,17 @@ function TestOperations(page) {
 
 	this.assertItemText = function (itemIndex, textToAssert) {
 		page.getItemLabelAtIndex(itemIndex).getText().then(function (text) {
-			assert.equal(textToAssert, text.trim());
+			assert.equal(textToAssert, text,
+				'A todo item with text \'' + textToAssert + '\' was expected at index ' +
+				itemIndex + ', the text \'' + text + '\' was observed');
 		});
 	};
 
 	// tests that the list contains the following items, independant of order
 	this.assertItems = function (textArray) {
 		page.getItemLabels().then(function (labels) {
-			assert.equal(textArray.length, labels.length);
+			assert.equal(textArray.length, labels.length,
+				textArray.length, + ' items expected in the todo list, ' + labels.length + ' items observed');
 			// create an array of promises which check the presence of the
 			// label text within the 'textArray'
 			var tests = [];
@@ -121,7 +126,7 @@ function TestOperations(page) {
 
 	this.assertItemCountText = function (textToAssert) {
 		page.getItemsCountElement().getText().then(function (text) {
-			assert.equal(textToAssert, text.trim());
+			assert.equal(textToAssert, text.trim(), 'the item count text was incorrect');
 		});
 	};
 
@@ -129,7 +134,8 @@ function TestOperations(page) {
 	this.assertItemAtIndexIsCompleted = function (index) {
 		page.getItemElements().then(function (toDoItems) {
 			toDoItems[index].getAttribute('class').then(function (cssClass) {
-				assert(cssClass.indexOf('completed') !== -1);
+				assert(cssClass.indexOf('completed') !== -1, 
+					'the item at index ' + index + ' should have been marked as completed');
 			});
 		});
 	};
@@ -139,7 +145,8 @@ function TestOperations(page) {
 			toDoItems[index].getAttribute('class').then(function (cssClass) {
 				// the maria implementation uses an 'incompleted' CSS class which is redundant
 				// TODO: this should really be moved into the pageLaxMode
-				assert(cssClass.indexOf('completed') === -1 || cssClass.indexOf('incompleted') !== -1);
+				assert(cssClass.indexOf('completed') === -1 || cssClass.indexOf('incompleted') !== -1,
+					'the item at index ' + index + ' should not have been marked as completed');
 			});
 		});
 	};
@@ -148,26 +155,33 @@ function TestOperations(page) {
 		return cssClass.indexOf('selected') !== -1;
 	}
 
-	this.assertFilterAtIndexIsSelected = function (index) {
+	this.assertFilterAtIndexIsSelected = function (selectedIndex) {
 		page.getFilterElements().then(function (filterElements) {
-			filterElements[0].getAttribute('class').then(function (cssClass) {
-				assert(index === 0 ? isSelected(cssClass) : !isSelected(cssClass));
-			});
 
-			filterElements[1].getAttribute('class').then(function (cssClass) {
-				assert(index === 1 ? isSelected(cssClass) : !isSelected(cssClass));
-			});
+			// create an array of promises, each one holding a test
+			var tests = [];
 
-			filterElements[2].getAttribute('class').then(function (cssClass) {
-				assert(index === 2 ? isSelected(cssClass) : !isSelected(cssClass));
-			});
+			// push a test into the array, avoiding the classic JS for loops + closures issue!
+			function pushTest(itemIndex) {
+				tests.push(filterElements[itemIndex].getAttribute('class').then(function (cssClass) {
+					assert(selectedIndex === itemIndex ? isSelected(cssClass) : !isSelected(cssClass),
+						'the filter / route at index ' + selectedIndex + ' should have been selected');
+				}));
+			}
+
+			for (var i=0; i<3;i++) {
+				pushTest(i);
+			};
+
+			// execute all the tests
+			return Q.all(tests);
 		});
 	};
 
 	this.assertCompleteAllIsClear = function () {
 		page.getMarkAllCompletedCheckBox().then(function (markAllCompleted) {
 			markAllCompleted.isSelected().then(function (isSelected) {
-				assert(!isSelected);
+				assert(!isSelected, 'the mark-all-completed checkbox should be clear');
 			});
 		});
 	};
@@ -175,7 +189,7 @@ function TestOperations(page) {
 	this.assertCompleteAllIsChecked = function () {
 		page.getMarkAllCompletedCheckBox().then(function (markAllCompleted) {
 			markAllCompleted.isSelected().then(function (isSelected) {
-				assert(isSelected);
+				assert(isSelected, 'the mark-all-completed checkbox should be checked');
 			});
 		});
 	};