Add fixture pausing and --open option to replace --manual

bin/test.js

@@ -0,0 +1,160 @@
+#!/usr/bin/env node
+import { ConfigLoaderError, readConfig } from '@web/config-loader';
+import commandLineArgs from 'command-line-args';
+import commandLineUsage from 'command-line-usage';
+import process from 'node:process';
+import { startTestRunner } from '@web/test-runner';
+import { WTRConfig } from '../src/server/wtr-config.js';
+
+const DISALLOWED_OPTIONS = ['--browsers', '--playwright', '--puppeteer', '--groups', '--manual'];
+
+const optionDefinitions = [
+	// @web/test-runner options
+	{
+		name: 'files',
+		type: String,
+		multiple: true,
+		description: 'Test files to run. Path or glob.\n[Default: ./test/**/*.<group>.js]',
+		order: 9
+	},
+	{
+		name: 'open',
+		type: Boolean,
+		description: 'Open the browser in headed mode'
+	},
+	{
+		name: 'slowmo',
+		type: Number,
+		description: 'Slows down test operations by the specified number of milliseconds. Useful so that you can see what is going on.'
+	},
+	{
+		name: 'group',
+		type: String,
+		required: true,
+		defaultOption: true,
+		description: 'Name of the group to run tests for\n[Default: test]',
+		order: 1
+	},
+	{
+		name: 'watch',
+		type: Boolean,
+		description: 'Reload tests on file changes. Allows debugging in all browsers.',
+		order: 10
+	},
+
+	// d2l-test options
+	{
+		name: 'chrome',
+		type: Boolean,
+		description: 'Run tests in Chromium',
+		order: 2
+	},
+	{
+		name: 'config',
+		alias: 'c',
+		type: String,
+		description: 'Location to read config file from\n[Default: ./d2l-test.config.js]',
+		order: 11
+	},
+	{
+		name: 'filter',
+		alias: 'f',
+		type: String,
+		multiple: true,
+		description: 'Filter test files by replacing wildcards with this glob',
+		order: 6
+	},
+	{
+		name: 'firefox',
+		type: Boolean,
+		description: 'Run tests in Firefox',
+		order: 3
+	},
+	{
+		name: 'golden',
+		type: Boolean,
+		description: 'Generate new golden screenshots',
+		order: 12
+	},
+	{
+		name: 'grep',
+		alias: 'g',
+		type: String,
+		description: 'Only run tests matching this string or regexp',
+		order: 7
+	},
+	{
+		name: 'help',
+		type: Boolean,
+		description: 'Print usage information and exit',
+		order: 13
+	},
+	{
+		name: 'retries',
+		type: Number,
+		description: 'Number of times to retry failed tests\n[Default: 0]',
+		order: 8
+	},
+	{
+		name: 'safari',
+		type: Boolean,
+		description: 'Run tests in Webkit',
+		order: 4
+	},
+	{
+		name: 'timeout',
+		alias: 't',
+		type: Number,
+		description: 'Test timeout threshold in ms\n[Default: 2000]',
+		order: 5
+	},
+];
+
+const cliArgs = commandLineArgs(optionDefinitions, { partial: true });
+
+if (cliArgs.help) {
+	const help = commandLineUsage([
+		{
+			header: 'D2L Test',
+			content: 'Test runner for D2L components and applications'
+		},
+		{
+			header: 'Usage',
+			content: 'd2l-test [options]',
+		},
+		{
+			header: 'Options',
+			optionList: optionDefinitions
+				.map(o => (o.description += '\n') && o)
+				.sort((a, b) => (a.order > b.order ? 1 : -1))
+		}
+	]);
+	process.stdout.write(help);
+	process.exit();
+}
+
+cliArgs._unknown = cliArgs._unknown?.filter(o => !DISALLOWED_OPTIONS.includes(o));
+
+const testConfig = await readConfig('d2l-test.config', cliArgs.config).catch(err => {
+	if (err instanceof ConfigLoaderError) {
+		throw new Error(err.message);
+	} else {
+		throw err;
+	}
+}) || {};
+
+const wtrConfig = new WTRConfig(cliArgs);
+const config = wtrConfig.create(testConfig);
+
+const argv = [
+	'--group', cliArgs.group,
+	...(cliArgs._unknown || [])
+];
+// copy cli-only wtr options back to argv to be processed
+cliArgs.watch && argv.push('--watch');
+
+await startTestRunner({
+	argv,
+	config,
+	readFileConfig: false
+});

package.json

@@ -10,9 +10,12 @@
     "test": "npm run lint && npm run test:server && npm run test:browser",
     "test:browser": "web-test-runner --files \"./test/browser/**/*.test.js\" --node-resolve --playwright",
     "test:server": "mocha ./test/server/**/*.test.js",
-    "test:vdiff": "web-test-runner --config ./test/browser/wtr-vdiff.config.js --group vdiff",
+    "test:vdiff": "d2l-test --config ./test/browser/d2l-test.config.js --group vdiff",
     "test:vdiff:golden": "npm run test:vdiff -- --golden"
   },
+  "bin": {
+    "d2l-test": "./bin/test.js"
+  },
   "author": "D2L Corporation",
   "license": "Apache-2.0",
   "devDependencies": {
@@ -30,8 +33,7 @@
     "sinon": "^15"
   },
   "exports": {
-    ".": "./src/browser/index.js",
-    "./wtr-config.js": "./src/server/index.js"
+    ".": "./src/browser/index.js"
   },
   "files": [
     "/src"
@@ -41,10 +43,12 @@
   },
   "dependencies": {
     "@open-wc/testing": "^3",
+    "@web/config-loader": "^0.2",
     "@web/test-runner": "^0.16",
     "@web/test-runner-commands": "^0.7",
     "@web/test-runner-playwright": "^0.10",
     "command-line-args": "^5",
+    "command-line-usage": "^7",
     "glob": "^10",
     "pixelmatch": "^5",
     "pngjs": "^7"

src/browser/commands.js

@@ -30,6 +30,7 @@ export async function focusElem(elem) {
 
 export async function hoverAt(x, y) {
 	await sendMouse({ type: 'move', position: [x, y] });
+	if (window.d2lTest) window.d2lTest.hovering = true;
 }
 
 export async function hoverElem(elem) {

src/browser/fixture.js

@@ -51,5 +51,18 @@ export async function fixture(element, opts = {}) {
 	await Promise.all([reset(opts), document.fonts.ready]);
 	const elem = await wcFixture(element);
 	await waitForElem(elem, opts.awaitLoadingComplete);
+
+	await pause();
 	return elem;
 }
+
+async function pause() {
+	const test = window.d2lTest || {};
+
+	test.update?.();
+
+	if (test.pause) {
+		await test.pause;
+		if (test.pause) test.pause = new Promise(r => test.run = r);
+	}
+}

src/browser/vdiff.js

@@ -1,28 +1,85 @@
-import { chai, expect } from '@open-wc/testing';
+import { chai, expect, nextFrame } from '@open-wc/testing';
 import { executeServerCommand } from '@web/test-runner-commands';
 
-let test;
+// start loading fonts early
+[...document.fonts].map(font => font.load());
 
-chai.Assertion.addMethod('golden', function(...args) {
-	return ScreenshotAndCompare.call({ test, elem: this._obj }, ...args); // eslint-disable-line no-invalid-this
+let test, soonPromise;
+
+/* eslint-disable no-undef, no-invalid-this */
+chai.Assertion.addMethod('golden', async function(...args) {
+	await soonPromise?.catch(err => expect.fail(err));
+	return ScreenshotAndCompare.call({ test, elem: this._obj }, ...args);
 });
-mocha.setup({ // eslint-disable-line no-undef
+
+mocha.setup({
 	rootHooks: {
 		beforeEach() {
 			test = this.currentTest;
 		}
 	}
 });
+/* eslint-enable */
 
 async function ScreenshotAndCompare(opts) {
+	if (window.d2lTest) {
+		inlineStyles(this.elem);
+		document.documentElement.classList.add('screenshot');
+	}
+
 	const name = this.test.fullTitle();
 	const rect = this.elem.getBoundingClientRect();
 	let result = await executeServerCommand('brightspace-visual-diff-compare', { name, rect, opts });
 	if (result.resizeRequired) {
 		this.test.timeout(0);
 		result = await executeServerCommand('brightspace-visual-diff-compare-resize', { name });
 	}
+
+	if (window.d2lTest) document.documentElement.classList.remove('screenshot');
+
 	if (!result.pass) {
+		if (window.d2lTest?.pause) {
+			window.d2lTest.fail();
+			await window.d2lTest.retryResponse;
+		}
 		expect.fail(result.message);
 	}
+
+	window.d2lTest?.pass();
+}
+
+const disallowedProps = ['width', 'inline-size'];
+let count = 0;
+function inlineStyles(elem) {
+	// headed chrome takes screenshots by first moving and locking down the element,
+	// which breaks the hover state. So, copy current styles inline before screenshot
+	if (!window.d2lTest.hovering || !window.chrome) return;
+
+	count += 1;
+
+	elem.classList.add(`__d2lTestHovering-${count}`);
+
+	[...elem.children, ...elem.shadowRoot?.children ?? []].forEach(child =>	inlineStyles(child));
+
+	const computedStyle = getComputedStyle(elem);
+	[...computedStyle].forEach(prop => {
+		if (!disallowedProps.includes(prop)) {
+			elem.style[prop] = computedStyle.getPropertyValue(prop);
+		}
+	});
+
+	['before', 'after'].forEach(pseudoEl => {
+		const computedStyle = getComputedStyle(elem, `::${pseudoEl}`);
+		if (computedStyle.content !== 'none') {
+			const sheet = new CSSStyleSheet();
+
+			const props = [...computedStyle].map(prop => {
+				const value = computedStyle.getPropertyValue(prop);
+				return disallowedProps.includes(prop) ? '' : `${prop}: ${value} !important;`;
+			}).join('');
+
+			sheet.insertRule(`.__d2lTestHovering-${count}::${pseudoEl} {${props}}`);
+			elem.getRootNode().adoptedStyleSheets.push(sheet);
+		}
+	});
 }

src/server/headed-mode-plugin.js

@@ -1,15 +1,17 @@
+import { dirname, join } from 'node:path';
 import { globSync } from 'glob';
 
-export function headedMode({ manual, watch, pattern }) {
+export function headedMode({ open, watch, pattern }) {
 
 	const files = globSync(pattern, { ignore: 'node_modules/**', posix: true });
 
 	return {
 		name: 'brightspace-headed-mode',
 		async transform(context) {
-			if ((watch || manual) && files.includes(context.path.slice(1))) {
-				watch && await new Promise(r => setTimeout(r, 2000));
-				return `debugger;\n${context.body}`;
+
+			if ((watch || open) && files.includes(context.path.slice(1))) {
+				const pausePath = join(dirname(import.meta.url), 'pause.js').replace('file:', '');
+				return `debugger;\nimport '${pausePath}'\n${context.body}`;
 			}
 		}
 	};